@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / bf3b79efb8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 00:45 UTC

[VLM] Qwen2.5-VL

Roger Wang committed Feb 5, 2025 at 13:31 UTC bf3b79efb82676219a3275764d8fcf4c70097ce5
14 files changed +1315 -52
docs/source/models/supported_models.md
+11
@@ -846,6 +846,13 @@ See [this page](#generative-models) for more information on how to use generativ
846 * ✅︎
847 * ✅︎
848 * ✅︎
849 +- * `Qwen2_5_VLForConditionalGeneration`
850 + * Qwen2.5-VL
851 + * T + I<sup>E+</sup> + V<sup>E+</sup>
852 + * `Qwen/Qwen2.5-VL-3B-Instruct`, `Qwen/Qwen2.5-VL-72B-Instruct`, etc.
853 + *
854 + * ✅︎
855 + * ✅︎
856 - * `UltravoxModel`
857 * Ultravox
858 * T + A<sup>E+</sup>
@@ -880,6 +887,10 @@ The chat template for Pixtral-HF is incorrect (see [discussion](https://huggingf
887 A corrected version is available at <gh-file:examples/template_pixtral_hf.jinja>.
888 :::
889
890 +:::{note}
891 +To use Qwen2.5-VL series models, you have to install Huggingface `transformers` library from source via `pip install git+https://github.com/huggingface/transformers`.
892 +:::
893 +
894 ### Pooling Models
895
896 See [this page](pooling-models) for more information on how to use pooling models.
examples/offline_inference/vision_language.py
+31
@@ -531,6 +531,36 @@ def run_qwen2_vl(question: str, modality: str):
531 return llm, prompt, stop_token_ids
532
533
534 +# Qwen2.5-VL
535 +def run_qwen2_5_vl(question: str, modality: str):
536 +
537 + model_name = "Qwen/Qwen2.5-VL-3B-Instruct"
538 +
539 + llm = LLM(
540 + model=model_name,
541 + max_model_len=4096,
542 + max_num_seqs=5,
543 + mm_processor_kwargs={
544 + "min_pixels": 28 * 28,
545 + "max_pixels": 1280 * 28 * 28,
546 + "fps": 1,
547 + },
548 + disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
549 + )
550 +
551 + if modality == "image":
552 + placeholder = "<|image_pad|>"
553 + elif modality == "video":
554 + placeholder = "<|video_pad|>"
555 +
556 + prompt = ("<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
557 + f"<|im_start|>user\n<|vision_start|>{placeholder}<|vision_end|>"
558 + f"{question}<|im_end|>\n"
559 + "<|im_start|>assistant\n")
560 + stop_token_ids = None
561 + return llm, prompt, stop_token_ids
562 +
563 +
564 model_example_map = {
565 "aria": run_aria,
566 "blip-2": run_blip2,
@@ -557,6 +587,7 @@ model_example_map = {
587 "pixtral_hf": run_pixtral_hf,
588 "qwen_vl": run_qwen_vl,
589 "qwen2_vl": run_qwen2_vl,
590 + "qwen2_5_vl": run_qwen2_5_vl,
591 }
592
593
examples/offline_inference/vision_language_multi_image.py
+58
@@ -392,6 +392,63 @@ def load_qwen2_vl(question, image_urls: List[str]) -> ModelRequestData:
392 )
393
394
395 +def load_qwen2_5_vl(question, image_urls: List[str]) -> ModelRequestData:
396 + try:
397 + from qwen_vl_utils import process_vision_info
398 + except ModuleNotFoundError:
399 + print('WARNING: `qwen-vl-utils` not installed, input images will not '
400 + 'be automatically resized. You can enable this functionality by '
401 + '`pip install qwen-vl-utils`.')
402 + process_vision_info = None
403 +
404 + model_name = "Qwen/Qwen2.5-VL-3B-Instruct"
405 +
406 + llm = LLM(
407 + model=model_name,
408 + max_model_len=32768 if process_vision_info is None else 4096,
409 + max_num_seqs=5,
410 + limit_mm_per_prompt={"image": len(image_urls)},
411 + )
412 +
413 + placeholders = [{"type": "image", "image": url} for url in image_urls]
414 + messages = [{
415 + "role": "system",
416 + "content": "You are a helpful assistant."
417 + }, {
418 + "role":
419 + "user",
420 + "content": [
421 + *placeholders,
422 + {
423 + "type": "text",
424 + "text": question
425 + },
426 + ],
427 + }]
428 +
429 + processor = AutoProcessor.from_pretrained(model_name)
430 +
431 + prompt = processor.apply_chat_template(messages,
432 + tokenize=False,
433 + add_generation_prompt=True)
434 +
435 + stop_token_ids = None
436 +
437 + if process_vision_info is None:
438 + image_data = [fetch_image(url) for url in image_urls]
439 + else:
440 + image_data, _ = process_vision_info(messages,
441 + return_video_sample_fps=False)
442 +
443 + return ModelRequestData(
444 + llm=llm,
445 + prompt=prompt,
446 + stop_token_ids=stop_token_ids,
447 + image_data=image_data,
448 + chat_template=None,
449 + )
450 +
451 +
452 model_example_map = {
453 "aria": load_aria,
454 "deepseek_vl_v2": load_deepseek_vl2,
@@ -404,6 +461,7 @@ model_example_map = {
461 "pixtral_hf": load_pixtral_hf,
462 "qwen_vl_chat": load_qwen_vl_chat,
463 "qwen2_vl": load_qwen2_vl,
464 + "qwen2_5_vl": load_qwen2_5_vl,
465 }
466
467
tests/models/decoder_only/vision_language/test_models.py
+22
@@ -121,6 +121,8 @@ VLM_TEST_SETTINGS = {
121 else ("half", "float")),
122 marks=[pytest.mark.core_model],
123 ),
124 + # TODO(ywang96): Move Qwen2-VL out of core models in favor of Qwen2.5-VL
125 + # once we upgraded to transformers>=4.49.0.
126 "qwen2_vl": VLMTestInfo(
127 models=["Qwen/Qwen2-VL-2B-Instruct"],
128 test_type=(
@@ -138,6 +140,26 @@ VLM_TEST_SETTINGS = {
140 image_size_factors=[(), (0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
141 marks=[pytest.mark.core_model, pytest.mark.cpu_model],
142 ),
143 + "qwen2_5_vl": VLMTestInfo(
144 + models=["Qwen/Qwen2.5-VL-3B-Instruct"],
145 + test_type=(
146 + VLMTestType.IMAGE,
147 + VLMTestType.MULTI_IMAGE,
148 + VLMTestType.VIDEO
149 + ),
150 + prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
151 + img_idx_to_prompt=lambda idx: "<|vision_start|><|image_pad|><|vision_end|>", # noqa: E501
152 + video_idx_to_prompt=lambda idx: "<|vision_start|><|video_pad|><|vision_end|>", # noqa: E501
153 + max_model_len=4096,
154 + max_num_seqs=2,
155 + auto_cls=AutoModelForVision2Seq,
156 + vllm_output_post_proc=model_utils.qwen2_vllm_to_hf_output,
157 + image_size_factors=[(), (0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
158 + marks=[pytest.mark.skipif(
159 + TRANSFORMERS_VERSION < "4.49.0",
160 + reason="HF model requires transformers>=4.49.0",
161 + ), pytest.mark.core_model, pytest.mark.cpu_model],
162 + ),
163 #### Extended model tests
164 "aria": VLMTestInfo(
165 models=["rhymes-ai/Aria"],
tests/models/multimodal/processing/test_common.py
+1
@@ -161,6 +161,7 @@ def _test_processing_correctness(
161 "nvidia/NVLM-D-72B",
162 "Qwen/Qwen-VL-Chat",
163 "Qwen/Qwen2-VL-2B-Instruct",
164 + "Qwen/Qwen2.5-VL-3B-Instruct",
165 "Qwen/Qwen2-Audio-7B-Instruct",
166 "fixie-ai/ultravox-v0_3",
167 ])
tests/models/registry.py
+2
@@ -264,6 +264,8 @@ _MULTIMODAL_EXAMPLE_MODELS = {
264 trust_remote_code=True),
265 "Qwen2AudioForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2-Audio-7B-Instruct"), # noqa: E501
266 "Qwen2VLForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2-VL-2B-Instruct"), # noqa: E501
267 + "Qwen2_5_VLForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2.5-VL-3B-Instruct", # noqa: E501
268 + min_transformers_version="4.49"), # noqa: E501
269 "UltravoxModel": _HfExamplesInfo("fixie-ai/ultravox-v0_3",
270 trust_remote_code=True),
271 # [Encoder-decoder]
vllm/entrypoints/chat_utils.py
+2 -2
@@ -410,7 +410,7 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
410 return "<image>"
411 if model_type == "mllama":
412 return "<|image|>"
413 - if model_type == "qwen2_vl":
413 + if model_type in ("qwen2_vl", "qwen2_5_vl"):
414 return "<|vision_start|><|image_pad|><|vision_end|>"
415 if model_type == "molmo":
416 return ""
@@ -430,7 +430,7 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
430 return "(<audio>./</audio>)"
431 raise TypeError(f"Unknown model type: {model_type}")
432 elif modality == "video":
433 - if model_type == "qwen2_vl":
433 + if model_type in ("qwen2_vl", "qwen2_5_vl"):
434 return "<|vision_start|><|video_pad|><|vision_end|>"
435 if model_type in ("minicpmo", "minicpmv"):
436 return "(<video>./</video>)"
vllm/model_executor/layers/rotary_embedding.py
+34 -24
@@ -27,6 +27,7 @@ from typing import Any, Dict, List, Optional, Tuple, Union
27
28 import torch
29 import torch.nn as nn
30 +from transformers import PretrainedConfig
31
32 from vllm.model_executor.custom_op import CustomOp
33
@@ -772,8 +773,12 @@ class MRotaryEmbedding(RotaryEmbedding):
773 dtype: torch.dtype,
774 mrope_section: Optional[List[int]] = None,
775 ) -> None:
775 - super().__init__(head_size, rotary_dim, max_position_embeddings, base,
776 - is_neox_style, dtype)
776 + # In Qwen2.5-VL, the maximum index value is related to the duration of
777 + # the input video. We enlarge max_position_embeddings to 4 times to get
778 + # a larger the cos and sin cache.
779 + self.cache_max_position_num = max_position_embeddings * 4
780 + super().__init__(head_size, rotary_dim, self.cache_max_position_num,
781 + base, is_neox_style, dtype)
782
783 self.mrope_section = mrope_section
784 if self.mrope_section:
@@ -831,13 +836,10 @@ class MRotaryEmbedding(RotaryEmbedding):
836 @staticmethod
837 def get_input_positions(
838 input_tokens: List[int],
839 + hf_config: PretrainedConfig,
840 image_grid_thw: Union[List[List[int]], torch.Tensor],
841 video_grid_thw: Union[List[List[int]], torch.Tensor],
836 - image_token_id: int,
837 - video_token_id: int,
838 - vision_start_token_id: int,
839 - vision_end_token_id: int,
840 - spatial_merge_size: int,
842 + second_per_grid_ts: Optional[List[float]] = None,
843 context_len: int = 0,
844 seq_len: Optional[int] = None,
845 ) -> Tuple[List[List[int]], int]:
@@ -845,16 +847,13 @@ class MRotaryEmbedding(RotaryEmbedding):
847
848 llm_positions, mrope_position_delta = \
849 MRotaryEmbedding.get_input_positions_tensor(
848 - input_tokens,
849 - image_grid_thw,
850 - video_grid_thw,
851 - image_token_id,
852 - video_token_id,
853 - vision_start_token_id,
854 - vision_end_token_id,
855 - spatial_merge_size,
856 - context_len,
857 - seq_len,
850 + input_tokens=input_tokens,
851 + hf_config=hf_config,
852 + image_grid_thw=image_grid_thw,
853 + video_grid_thw=video_grid_thw,
854 + second_per_grid_ts=second_per_grid_ts,
855 + context_len=context_len,
856 + seq_len=seq_len,
857 )
858
859 return llm_positions.tolist(), mrope_position_delta
@@ -862,18 +861,22 @@ class MRotaryEmbedding(RotaryEmbedding):
861 @staticmethod
862 def get_input_positions_tensor(
863 input_tokens: List[int],
864 + hf_config: PretrainedConfig,
865 image_grid_thw: Union[List[List[int]], torch.Tensor],
866 video_grid_thw: Union[List[List[int]], torch.Tensor],
867 - image_token_id: int,
868 - video_token_id: int,
869 - vision_start_token_id: int,
870 - vision_end_token_id: int,
871 - spatial_merge_size: int,
867 + second_per_grid_ts: Optional[List[float]] = None,
868 context_len: int = 0,
869 seq_len: Optional[int] = None,
870 ) -> Tuple[torch.Tensor, int]:
871 """Get mrope input positions and delta value."""
872
873 + image_token_id = hf_config.image_token_id
874 + video_token_id = hf_config.video_token_id
875 + vision_start_token_id = hf_config.vision_start_token_id
876 + spatial_merge_size = hf_config.vision_config.spatial_merge_size
877 + tokens_per_second = getattr(hf_config.vision_config,
878 + "tokens_per_second", 1.0)
879 +
880 if isinstance(image_grid_thw, torch.Tensor):
881 image_grid_thw = image_grid_thw.tolist()
882 if isinstance(video_grid_thw, torch.Tensor):
@@ -892,6 +895,7 @@ class MRotaryEmbedding(RotaryEmbedding):
895
896 image_index, video_index = 0, 0
897 for _ in range(image_nums + video_nums):
898 + video_second_per_grid_t = 0.0
899 if image_token_id in input_tokens and remain_images > 0:
900 ed_image = input_tokens.index(image_token_id, st)
901 else:
@@ -915,9 +919,13 @@ class MRotaryEmbedding(RotaryEmbedding):
919 video_grid_thw[video_index][1],
920 video_grid_thw[video_index][2],
921 )
922 + video_second_per_grid_t = 1.0
923 + if second_per_grid_ts is not None:
924 + video_second_per_grid_t = second_per_grid_ts[video_index]
925 video_index += 1
926 remain_videos -= 1
927 ed = ed_video
928 +
929 llm_grid_t, llm_grid_h, llm_grid_w = \
930 t, h // spatial_merge_size, w // spatial_merge_size
931 text_len = ed - st
@@ -927,8 +935,10 @@ class MRotaryEmbedding(RotaryEmbedding):
935 llm_pos_ids_list.append(
936 torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx)
937
930 - t_index = torch.arange(llm_grid_t).view(-1, 1).expand(
931 - -1, llm_grid_h * llm_grid_w).flatten()
938 + t_index = (torch.arange(llm_grid_t).view(-1, 1).expand(
939 + -1, llm_grid_h * llm_grid_w) * video_second_per_grid_t *
940 + tokens_per_second).long().flatten()
941 +
942 h_index = torch.arange(llm_grid_h).view(1, -1, 1).expand(
943 llm_grid_t, -1, llm_grid_w).flatten()
944 w_index = torch.arange(llm_grid_w).view(1, 1, -1).expand(
vllm/model_executor/models/qwen2_5_vl.py new
+1133
@@ -0,0 +1,1133 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +
3 +# Adapted from
4 +# https://github.com/huggingface/transformers/blob/main/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py
5 +# Copyright 2025 The vLLM team.
6 +# Copyright 2025 The Qwen Team.
7 +# Copyright 2025 The HuggingFace Inc. team.
8 +# All rights reserved.
9 +#
10 +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
11 +# and OPT implementations in this library. It has been modified from its
12 +# original forms to accommodate minor architectural differences compared
13 +# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
14 +#
15 +# Licensed under the Apache License, Version 2.0 (the "License");
16 +# you may not use this file except in compliance with the License.
17 +# You may obtain a copy of the License at
18 +#
19 +# http://www.apache.org/licenses/LICENSE-2.0
20 +#
21 +# Unless required by applicable law or agreed to in writing, software
22 +# distributed under the License is distributed on an "AS IS" BASIS,
23 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
24 +# See the License for the specific language governing permissions and
25 +# limitations under the License.
26 +"""Inference-only Qwen2.5-VL model compatible with HuggingFace weights."""
27 +from functools import cached_property, partial
28 +from typing import (Callable, Iterable, List, Literal, Mapping, Optional, Set,
29 + Tuple, TypedDict, Union)
30 +
31 +import torch
32 +import torch.nn as nn
33 +import torch.nn.functional as F
34 +from einops import rearrange
35 +from transformers import BatchFeature
36 +from transformers.models.qwen2_5_vl import (Qwen2_5_VLImageProcessor,
37 + Qwen2_5_VLProcessor)
38 +from transformers.models.qwen2_5_vl.configuration_qwen2_5_vl import (
39 + Qwen2_5_VLConfig, Qwen2_5_VLVisionConfig)
40 +
41 +from vllm.attention import AttentionMetadata
42 +from vllm.config import VllmConfig
43 +from vllm.distributed import parallel_state
44 +from vllm.distributed import utils as dist_utils
45 +from vllm.logger import init_logger
46 +from vllm.model_executor import SamplingMetadata
47 +from vllm.model_executor.layers.activation import _ACTIVATION_REGISTRY
48 +from vllm.model_executor.layers.linear import (ColumnParallelLinear,
49 + RowParallelLinear)
50 +from vllm.model_executor.layers.quantization import QuantizationConfig
51 +from vllm.model_executor.layers.quantization.gptq import GPTQConfig
52 +from vllm.model_executor.layers.quantization.gptq_marlin import (
53 + GPTQMarlinConfig)
54 +from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
55 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
56 +from vllm.model_executor.models.module_mapping import MultiModelKeys
57 +from vllm.multimodal import MULTIMODAL_REGISTRY
58 +from vllm.multimodal.inputs import MultiModalFieldConfig
59 +from vllm.platforms import _Backend
60 +from vllm.sequence import IntermediateTensors
61 +from vllm.transformers_utils.config import uses_mrope
62 +
63 +from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP
64 +from .qwen2_vl import Qwen2VLDummyInputsBuilder as Qwen2_5_VLDummyInputsBuilder
65 +from .qwen2_vl import (Qwen2VLMultiModalProcessor, Qwen2VLProcessingInfo,
66 + apply_rotary_pos_emb_vision)
67 +from .utils import (AutoWeightsLoader, WeightsMapper,
68 + init_vllm_registered_model, maybe_prefix,
69 + merge_multimodal_embeddings)
70 +from .vision import get_vit_attn_backend
71 +
72 +logger = init_logger(__name__)
73 +
74 +# === Vision Inputs === #
75 +
76 +
77 +class Qwen2_5_VLImagePixelInputs(TypedDict):
78 + type: Literal["pixel_values"]
79 + pixel_values: torch.Tensor
80 + """Shape:
81 + `(num_patches, num_channels * patch_size * patch_size)`
82 + """
83 +
84 + image_grid_thw: torch.Tensor
85 + """Shape: `(num_images, 3)`
86 + This should be in `(grid_t, grid_h, grid_w)` format.
87 + """
88 +
89 +
90 +class Qwen2_5_VLImageEmbeddingInputs(TypedDict):
91 + type: Literal["image_embeds"]
92 + image_embeds: torch.Tensor
93 + """Supported types:
94 + - List[`torch.Tensor`]: A list of tensors holding all images' features.
95 + Each tensor holds an image's features.
96 + - `torch.Tensor`: A tensor holding all images' features
97 + (concatenation of all images' feature tensors).
98 +
99 + Tensor shape: `(num_image_features, hidden_size)`
100 + - `num_image_features` varies based on
101 + the number and resolution of the images.
102 + - `hidden_size` must match the hidden size of language model backbone.
103 + """
104 +
105 + image_grid_thw: torch.Tensor
106 + """Shape: `(num_images, 3)`
107 + This should be in `(grid_t, grid_h, grid_w)` format.
108 + """
109 +
110 +
111 +Qwen2_5_VLImageInputs = Union[Qwen2_5_VLImagePixelInputs,
112 + Qwen2_5_VLImageEmbeddingInputs]
113 +
114 +
115 +class Qwen2_5_VLVideoPixelInputs(TypedDict):
116 + type: Literal["pixel_values_videos"]
117 + pixel_values_videos: torch.Tensor
118 + """Shape:
119 + `(num_patches,
120 + num_channels * temporal_patch_size * patch_size * patch_size)`
121 + """
122 +
123 + video_grid_thw: torch.Tensor
124 + """Shape: `(num_videos, 3)`
125 +
126 + This should be in `(grid_t, grid_h, grid_w)` format.
127 + """
128 +
129 + second_per_grid_ts: torch.Tensor
130 + """
131 + The video time interval (in seconds) for each grid along the temporal
132 + dimension in the 3D position IDs. Returned when `videos` is not `None`.
133 + """
134 +
135 +
136 +class Qwen2_5_VLVideoEmbeddingInputs(TypedDict):
137 + type: Literal["video_embeds"]
138 + video_embeds: torch.Tensor
139 + """Supported types:
140 + - List[`torch.Tensor`]: A list of tensors holding all videos' features.
141 + Each tensor holds an video's features.
142 + - `torch.Tensor`: A tensor holding all videos' features
143 + (concatenation of all videos' feature tensors).
144 +
145 + Tensor shape: `(num_image_features, hidden_size)`
146 + - `num_image_features` varies based on
147 + the number and resolution of the videos.
148 + - `hidden_size` must match the hidden size of language model backbone.
149 + """
150 +
151 + video_grid_thw: torch.Tensor
152 + """Shape: `(num_videos, 3)`
153 + This should be in `(grid_t, grid_h, grid_w)` format.
154 + """
155 +
156 +
157 +Qwen2_5_VLVideoInputs = Union[Qwen2_5_VLVideoPixelInputs,
158 + Qwen2_5_VLVideoEmbeddingInputs]
159 +
160 +# === Vision Encoder === #
161 +
162 +
163 +class Qwen2_5_VisionMLP(nn.Module):
164 +
165 + def __init__(self,
166 + in_features: int,
167 + hidden_features: int,
168 + bias: bool = False,
169 + act_fn: Callable[[torch.Tensor], torch.Tensor] = F.silu,
170 + quant_config: Optional[QuantizationConfig] = None,
171 + prefix: str = ""):
172 + super().__init__()
173 + self.gate_proj = ColumnParallelLinear(in_features,
174 + hidden_features,
175 + bias=bias,
176 + quant_config=quant_config,
177 + prefix=f"{prefix}.gate_proj")
178 + self.up_proj = ColumnParallelLinear(in_features,
179 + hidden_features,
180 + bias=bias,
181 + quant_config=quant_config,
182 + prefix=f"{prefix}.up_proj")
183 + self.down_proj = RowParallelLinear(hidden_features,
184 + in_features,
185 + bias=bias,
186 + quant_config=quant_config,
187 + prefix=f"{prefix}.down_proj")
188 + self.act_fn = act_fn
189 +
190 + def forward(self, x: torch.Tensor):
191 + x_gate, _ = self.gate_proj(x)
192 + x_gate = self.act_fn(x_gate)
193 + x_up, _ = self.up_proj(x)
194 + x_down, _ = self.down_proj(x_gate * x_up)
195 + return x_down
196 +
197 +
198 +class Qwen2_5_VisionAttention(nn.Module):
199 +
200 + def __init__(
201 + self,
202 + embed_dim: int,
203 + num_heads: int,
204 + projection_size: int,
205 + quant_config: Optional[QuantizationConfig] = None,
206 + prefix: str = "",
207 + ) -> None:
208 + super().__init__()
209 + # Per attention head and per partition values.
210 + world_size = parallel_state.get_tensor_model_parallel_world_size()
211 + self.hidden_size_per_attention_head = dist_utils.divide(
212 + projection_size, num_heads)
213 + self.num_attention_heads_per_partition = dist_utils.divide(
214 + num_heads, world_size)
215 +
216 + self.qkv = ColumnParallelLinear(input_size=embed_dim,
217 + output_size=3 * projection_size,
218 + quant_config=quant_config,
219 + prefix=f"{prefix}.qkv")
220 + self.proj = RowParallelLinear(input_size=projection_size,
221 + output_size=embed_dim,
222 + quant_config=quant_config,
223 + prefix=f"{prefix}.proj")
224 +
225 + # Detect attention implementation.
226 + self.attn_backend: _Backend = get_vit_attn_backend(support_fa=True)
227 + if self.attn_backend not in {
228 + _Backend.FLASH_ATTN, _Backend.TORCH_SDPA, _Backend.XFORMERS
229 + }:
230 + raise RuntimeError(
231 + f"Qwen2.5-VL does not support {self.attn_backend} backend now."
232 + )
233 +
234 + def forward(
235 + self,
236 + x: torch.Tensor,
237 + cu_seqlens: torch.Tensor,
238 + rotary_pos_emb: torch.Tensor,
239 + ) -> torch.Tensor:
240 + # [s, b, c] --> [s, b, head * 3 * head_dim]
241 + x, _ = self.qkv(x)
242 +
243 + # [s, b, head * 3 * head_dim] --> [s, b, head, 3 * head_dim]
244 + new_x_shape = x.size()[:-1] + (
245 + self.num_attention_heads_per_partition,
246 + 3 * self.hidden_size_per_attention_head,
247 + )
248 + x = x.view(*new_x_shape)
249 +
250 + # [s, b, head, 3 * head_dim] --> 3 [s, b, head, head_dim]
251 + q, k, v = dist_utils.split_tensor_along_last_dim(x, 3)
252 + batch_size = q.shape[1]
253 +
254 + q, k, v = (rearrange(x, "s b ... -> b s ...").contiguous()
255 + for x in (q, k, v))
256 + if rotary_pos_emb is not None:
257 + q = apply_rotary_pos_emb_vision(q, rotary_pos_emb)
258 + k = apply_rotary_pos_emb_vision(k, rotary_pos_emb)
259 +
260 + if self.attn_backend == _Backend.FLASH_ATTN:
261 + # from vllm_flash_attn.flash_attn_interface import (
262 + # flash_attn_varlen_func)
263 + from flash_attn import flash_attn_varlen_func
264 +
265 + q, k, v = (rearrange(x, "b s ... -> (b s) ...") for x in [q, k, v])
266 +
267 + max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max().item()
268 + output = flash_attn_varlen_func(q,
269 + k,
270 + v,
271 + cu_seqlens_q=cu_seqlens,
272 + cu_seqlens_k=cu_seqlens,
273 + max_seqlen_q=max_seqlen,
274 + max_seqlen_k=max_seqlen,
275 + dropout_p=0,
276 + causal=False)
277 +
278 + context_layer = rearrange(output,
279 + "(b s) ... -> b s ...",
280 + b=batch_size)
281 + elif self.attn_backend == _Backend.TORCH_SDPA:
282 + seq_length = q.size(1)
283 + q, k, v = (rearrange(x, "b s h d -> b h s d") for x in [q, k, v])
284 + attention_mask = torch.zeros([1, seq_length, seq_length],
285 + device=q.device,
286 + dtype=torch.bool)
287 + for i in range(1, len(cu_seqlens)):
288 + attention_mask[..., cu_seqlens[i - 1]:cu_seqlens[i],
289 + cu_seqlens[i - 1]:cu_seqlens[i]] = True
290 + output = F.scaled_dot_product_attention(q,
291 + k,
292 + v,
293 + attention_mask,
294 + dropout_p=0.0)
295 + context_layer = rearrange(output, "b h s d -> b s h d ")
296 + elif self.attn_backend == _Backend.XFORMERS:
297 + from xformers import ops as xops
298 + from xformers.ops.fmha.attn_bias import BlockDiagonalMask
299 +
300 + seqlens = (cu_seqlens[1:] - cu_seqlens[:-1]).tolist()
301 + attn_bias = BlockDiagonalMask.from_seqlens(q_seqlen=seqlens,
302 + kv_seqlen=None)
303 +
304 + context_layer = xops.memory_efficient_attention_forward(
305 + q, k, v, attn_bias=attn_bias, p=0, scale=None)
306 + context_layer = rearrange(context_layer,
307 + "b s h d -> s b (h d)").contiguous()
308 +
309 + output, _ = self.proj(context_layer)
310 + return output
311 +
312 +
313 +class Qwen2RMSNorm(nn.Module):
314 +
315 + def __init__(self, hidden_size, eps=1e-6):
316 + super().__init__()
317 + self.weight = nn.Parameter(torch.ones(hidden_size))
318 + self.variance_epsilon = eps
319 +
320 + def forward(self, hidden_states):
321 + input_dtype = hidden_states.dtype
322 + hidden_states = hidden_states.to(torch.float32)
323 + variance = hidden_states.pow(2).mean(-1, keepdim=True)
324 + hidden_states = hidden_states * torch.rsqrt(variance +
325 + self.variance_epsilon)
326 + return self.weight * hidden_states.to(input_dtype)
327 +
328 + def extra_repr(self):
329 + return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}"
330 +
331 +
332 +class Qwen2_5_VisionBlock(nn.Module):
333 +
334 + def __init__(
335 + self,
336 + dim: int,
337 + num_heads: int,
338 + mlp_hidden_dim: int,
339 + act_fn: Callable[[torch.Tensor], torch.Tensor] = F.silu,
340 + norm_layer: Optional[Callable[[int], nn.Module]] = None,
341 + quant_config: Optional[QuantizationConfig] = None,
342 + prefix: str = "",
343 + ) -> None:
344 + super().__init__()
345 + if norm_layer is None:
346 + norm_layer = partial(nn.LayerNorm, eps=1e-6)
347 + self.norm1 = norm_layer(dim)
348 + self.norm2 = norm_layer(dim)
349 + self.attn = Qwen2_5_VisionAttention(embed_dim=dim,
350 + num_heads=num_heads,
351 + projection_size=dim,
352 + quant_config=quant_config,
353 + prefix=f"{prefix}.attn")
354 + self.mlp = Qwen2_5_VisionMLP(dim,
355 + mlp_hidden_dim,
356 + act_fn=act_fn,
357 + bias=True,
358 + quant_config=quant_config,
359 + prefix=f"{prefix}.mlp")
360 +
361 + def forward(self, x: torch.Tensor, cu_seqlens: torch.Tensor,
362 + rotary_pos_emb: torch.Tensor) -> torch.Tensor:
363 + x = x + self.attn(self.norm1(x),
364 + cu_seqlens=cu_seqlens,
365 + rotary_pos_emb=rotary_pos_emb)
366 + x = x + self.mlp(self.norm2(x))
367 + return x
368 +
369 +
370 +class Qwen2_5_VisionPatchEmbed(nn.Module):
371 +
372 + def __init__(
373 + self,
374 + patch_size: int = 14,
375 + temporal_patch_size: int = 2,
376 + in_channels: int = 3,
377 + hidden_size: int = 1152,
378 + ) -> None:
379 + super().__init__()
380 + self.patch_size = patch_size
381 + self.temporal_patch_size = temporal_patch_size
382 + self.hidden_size = hidden_size
383 +
384 + kernel_size = (temporal_patch_size, patch_size, patch_size)
385 + self.proj = nn.Conv3d(in_channels,
386 + hidden_size,
387 + kernel_size=kernel_size,
388 + stride=kernel_size,
389 + bias=False)
390 +
391 + def forward(self, x: torch.Tensor) -> torch.Tensor:
392 + L, C = x.shape
393 + x = x.view(L, -1, self.temporal_patch_size, self.patch_size,
394 + self.patch_size)
395 + x = self.proj(x).view(L, self.hidden_size)
396 + return x
397 +
398 +
399 +class Qwen2_5_VisionPatchMerger(nn.Module):
400 +
401 + def __init__(
402 + self,
403 + d_model: int,
404 + context_dim: int,
405 + norm_layer: Optional[Callable[[int], nn.Module]] = None,
406 + spatial_merge_size: int = 2,
407 + quant_config: Optional[QuantizationConfig] = None,
408 + prefix: str = "",
409 + ) -> None:
410 + super().__init__()
411 + self.hidden_size = context_dim * (spatial_merge_size**2)
412 + if norm_layer is None:
413 + norm_layer = partial(nn.LayerNorm, eps=1e-6)
414 + self.ln_q = norm_layer(context_dim)
415 + self.mlp = nn.ModuleList([
416 + ColumnParallelLinear(self.hidden_size,
417 + self.hidden_size,
418 + bias=True,
419 + quant_config=quant_config,
420 + prefix=f"{prefix}.mlp.0"),
421 + nn.GELU(),
422 + RowParallelLinear(self.hidden_size,
423 + d_model,
424 + bias=True,
425 + quant_config=quant_config,
426 + prefix=f"{prefix}.mlp.2"),
427 + ])
428 +
429 + def forward(self, x: torch.Tensor) -> torch.Tensor:
430 + x = self.ln_q(x)
431 + x = x.view(-1, self.hidden_size)
432 +
433 + mlp_fc1, mlp_act, mlp_fc2 = self.mlp
434 + x_parallel, _ = mlp_fc1(x)
435 + x_parallel = mlp_act(x_parallel)
436 + out, _ = mlp_fc2(x_parallel)
437 + return out
438 +
439 +
440 +class Qwen2_5_VisionRotaryEmbedding(nn.Module):
441 +
442 + def __init__(self, dim: int, theta: float = 10000.0) -> None:
443 + super().__init__()
444 + self.dim = dim
445 + self.theta = theta
446 + inv_freq = 1.0 / (theta
447 + **(torch.arange(0, dim, 2, dtype=torch.float) / dim))
448 + self.register_buffer("inv_freq", inv_freq, persistent=False)
449 + self._seq_len_cached = 0
450 + self._freqs_cached = None
451 +
452 + def update_freqs_cache(self, seqlen: int) -> None:
453 + if seqlen > self._seq_len_cached:
454 + seqlen *= 2
455 + self._seq_len_cached = seqlen
456 + self.inv_freq = 1.0 / (self.theta**(torch.arange(
457 + 0, self.dim, 2, dtype=torch.float, device=self.inv_freq.device)
458 + / self.dim))
459 + seq = torch.arange(seqlen,
460 + device=self.inv_freq.device,
461 + dtype=self.inv_freq.dtype)
462 + freqs = torch.outer(seq, self.inv_freq)
463 + self._freqs_cached = freqs
464 +
465 + def forward(self, seqlen: int) -> torch.Tensor:
466 + self.update_freqs_cache(seqlen)
467 + return self._freqs_cached[:seqlen]
468 +
469 +
470 +class Qwen2_5_VisionTransformer(nn.Module):
471 +
472 + def __init__(
473 + self,
474 + vision_config: Qwen2_5_VLVisionConfig,
475 + norm_eps: float = 1e-6,
476 + quant_config: Optional[QuantizationConfig] = None,
477 + prefix: str = "",
478 + ) -> None:
479 + super().__init__()
480 +
481 + patch_size = vision_config.patch_size
482 + temporal_patch_size = vision_config.temporal_patch_size
483 + in_channels = vision_config.in_channels
484 + depth = vision_config.depth
485 + self.hidden_size = vision_config.hidden_size
486 + self.num_heads = vision_config.num_heads
487 +
488 + # args for get_window_index
489 + self.window_size = vision_config.window_size
490 + self.patch_size = vision_config.patch_size
491 + self.spatial_merge_size = vision_config.spatial_merge_size
492 + self.fullatt_block_indexes = vision_config.fullatt_block_indexes
493 + self.spatial_merge_unit = self.spatial_merge_size**2
494 +
495 + self.patch_embed = Qwen2_5_VisionPatchEmbed(
496 + patch_size=patch_size,
497 + temporal_patch_size=temporal_patch_size,
498 + in_channels=in_channels,
499 + hidden_size=self.hidden_size,
500 + )
501 +
502 + # NOTE: We use torch native RMSNorm here for precision purposes.
503 + norm_layer = partial(Qwen2RMSNorm, eps=norm_eps)
504 + head_dim = self.hidden_size // self.num_heads
505 + self.rotary_pos_emb = Qwen2_5_VisionRotaryEmbedding(head_dim // 2)
506 +
507 + self.blocks = nn.ModuleList([
508 + Qwen2_5_VisionBlock(
509 + dim=self.hidden_size,
510 + num_heads=self.num_heads,
511 + mlp_hidden_dim=vision_config.intermediate_size,
512 + act_fn=_ACTIVATION_REGISTRY[vision_config.hidden_act],
513 + norm_layer=norm_layer,
514 + quant_config=quant_config,
515 + prefix=f"{prefix}.blocks.{layer_idx}")
516 + for layer_idx in range(depth)
517 + ])
518 + self.merger = Qwen2_5_VisionPatchMerger(
519 + d_model=vision_config.out_hidden_size,
520 + context_dim=self.hidden_size,
521 + norm_layer=norm_layer,
522 + spatial_merge_size=self.spatial_merge_size,
523 + quant_config=quant_config,
524 + prefix=f"{prefix}.merger",
525 + )
526 +
527 + @property
528 + def dtype(self) -> torch.dtype:
529 + return self.patch_embed.proj.weight.dtype
530 +
531 + @property
532 + def device(self) -> torch.device:
533 + return self.patch_embed.proj.weight.device
534 +
535 + def rot_pos_emb(self, grid_thw: torch.Tensor) -> torch.Tensor:
536 + pos_ids = []
537 + for t, h, w in grid_thw:
538 + hpos_ids = torch.arange(h).unsqueeze(1).expand(-1, w)
539 + wpos_ids = torch.arange(w).unsqueeze(0).expand(h, -1)
540 + hpos_ids = hpos_ids.reshape(
541 + h // self.spatial_merge_size,
542 + self.spatial_merge_size,
543 + w // self.spatial_merge_size,
544 + self.spatial_merge_size,
545 + ).permute(0, 2, 1, 3).flatten()
546 + wpos_ids = wpos_ids.reshape(
547 + h // self.spatial_merge_size,
548 + self.spatial_merge_size,
549 + w // self.spatial_merge_size,
550 + self.spatial_merge_size,
551 + ).permute(0, 2, 1, 3).flatten()
552 + pos_ids.append(
553 + torch.stack([hpos_ids, wpos_ids], dim=-1).repeat(t, 1))
554 + pos_ids = torch.cat(pos_ids, dim=0)
555 + max_grid_size = grid_thw[:, 1:].max()
556 + rotary_pos_emb_full = self.rotary_pos_emb(max_grid_size)
557 + rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1)
558 + return rotary_pos_emb
559 +
560 + def get_window_index(self, grid_thw):
561 + window_index: list = []
562 + cu_window_seqlens: list = [0]
563 + window_index_id = 0
564 + vit_merger_window_size = (self.window_size //
565 + self.spatial_merge_size // self.patch_size)
566 +
567 + for grid_t, grid_h, grid_w in grid_thw:
568 + llm_grid_h = grid_h // self.spatial_merge_size
569 + llm_grid_w = grid_w // self.spatial_merge_size
570 + index = torch.arange(grid_t * llm_grid_h * llm_grid_w).reshape(
571 + grid_t, llm_grid_h, llm_grid_w)
572 + pad_h = vit_merger_window_size - llm_grid_h % vit_merger_window_size
573 + pad_w = vit_merger_window_size - llm_grid_w % vit_merger_window_size
574 + num_windows_h = (llm_grid_h + pad_h) // vit_merger_window_size
575 + num_windows_w = (llm_grid_w + pad_w) // vit_merger_window_size
576 + index_padded = F.pad(index, (0, pad_w, 0, pad_h), 'constant', -100)
577 + index_padded = index_padded.reshape(grid_t, num_windows_h,
578 + vit_merger_window_size,
579 + num_windows_w,
580 + vit_merger_window_size)
581 + index_padded = index_padded.permute(0, 1, 3, 2, 4).reshape(
582 + grid_t, num_windows_h * num_windows_w, vit_merger_window_size,
583 + vit_merger_window_size)
584 + seqlens = (index_padded != -100).sum([2, 3]).reshape(-1)
585 + index_padded = index_padded.reshape(-1)
586 + index_new = index_padded[index_padded != -100]
587 + window_index.append(index_new + window_index_id)
588 + cu_seqlens_tmp = seqlens.cumsum(
589 + 0) * self.spatial_merge_unit + cu_window_seqlens[-1]
590 + cu_window_seqlens.extend(cu_seqlens_tmp.tolist())
591 + window_index_id += (grid_t * llm_grid_h * llm_grid_w).item()
592 + window_index = torch.cat(window_index, dim=0)
593 + return window_index, cu_window_seqlens
594 +
595 + def forward(
596 + self,
597 + x: torch.Tensor,
598 + grid_thw: torch.Tensor,
599 + ) -> torch.Tensor:
600 + # patchify
601 + hidden_states = x.to(device=self.device, dtype=self.dtype)
602 + hidden_states = self.patch_embed(hidden_states)
603 +
604 + # compute position embedding
605 + rotary_pos_emb = self.rot_pos_emb(grid_thw)
606 +
607 + # windows attention
608 + window_index, cu_window_seqlens = self.get_window_index(grid_thw)
609 + cu_window_seqlens = torch.tensor(
610 + cu_window_seqlens,
611 + device=hidden_states.device,
612 + dtype=grid_thw.dtype if torch.jit.is_tracing() else torch.int32)
613 + cu_window_seqlens = torch.unique_consecutive(cu_window_seqlens)
614 + seq_len, _ = hidden_states.size()
615 + hidden_states = hidden_states.reshape(
616 + seq_len // self.spatial_merge_unit, self.spatial_merge_unit, -1)
617 + hidden_states = hidden_states[window_index, :, :]
618 + hidden_states = hidden_states.reshape(seq_len, -1)
619 + rotary_pos_emb = rotary_pos_emb.reshape(
620 + seq_len // self.spatial_merge_unit, self.spatial_merge_unit, -1)
621 + rotary_pos_emb = rotary_pos_emb[window_index, :, :]
622 + rotary_pos_emb = rotary_pos_emb.reshape(seq_len, -1)
623 + # compute cu_seqlens
624 + cu_seqlens = torch.repeat_interleave(grid_thw[:, 1] * grid_thw[:, 2],
625 + grid_thw[:, 0]).cumsum(
626 + dim=0, dtype=torch.int32)
627 + cu_seqlens = F.pad(cu_seqlens, (1, 0), "constant", 0)
628 +
629 + # transformers
630 + hidden_states = hidden_states.unsqueeze(1)
631 + for layer_num, blk in enumerate(self.blocks):
632 + if layer_num in self.fullatt_block_indexes:
633 + cu_seqlens_now = cu_seqlens
634 + else:
635 + cu_seqlens_now = cu_window_seqlens
636 + hidden_states = blk(hidden_states,
637 + cu_seqlens=cu_seqlens_now,
638 + rotary_pos_emb=rotary_pos_emb)
639 +
640 + # adapter
641 + hidden_states = self.merger(hidden_states)
642 + reverse_indices = torch.argsort(window_index)
643 + hidden_states = hidden_states[reverse_indices, :]
644 + return hidden_states
645 +
646 + def load_weights(self, weights: Iterable[Tuple[str,
647 + torch.Tensor]]) -> Set[str]:
648 + stacked_params_mapping = [
649 + # (param_name, shard_name, shard_id)
650 + ("qkv_proj", "q_proj", "q"),
651 + ("qkv_proj", "k_proj", "k"),
652 + ("qkv_proj", "v_proj", "v"),
653 + ]
654 + params_dict = dict(self.named_parameters(remove_duplicate=False))
655 + loaded_params: Set[str] = set()
656 +
657 + for name, loaded_weight in weights:
658 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
659 + if weight_name not in name:
660 + continue
661 + name = name.replace(weight_name, param_name)
662 +
663 + param = params_dict[name]
664 + weight_loader = param.weight_loader
665 + weight_loader(param, loaded_weight, shard_id)
666 + break
667 + else:
668 + if name.endswith("qkv.weight"):
669 + visual_num_heads = self.num_heads
670 + visual_embed_dim = self.hidden_size
671 + head_size = visual_embed_dim // visual_num_heads
672 + loaded_weight = loaded_weight.view(3, visual_num_heads,
673 + head_size,
674 + visual_embed_dim)
675 + loaded_weight = loaded_weight.transpose(0, 1)
676 + loaded_weight = loaded_weight.reshape(-1, visual_embed_dim)
677 + elif name.endswith("qkv.bias"):
678 + visual_num_heads = self.num_heads
679 + visual_embed_dim = self.hidden_size
680 + head_size = visual_embed_dim // visual_num_heads
681 + loaded_weight = loaded_weight.view(3, visual_num_heads,
682 + head_size)
683 + loaded_weight = loaded_weight.transpose(0, 1)
684 + loaded_weight = loaded_weight.reshape(-1)
685 +
686 + param = params_dict[name]
687 + weight_loader = getattr(param, "weight_loader",
688 + default_weight_loader)
689 + weight_loader(param, loaded_weight)
690 + loaded_params.add(name)
691 + return loaded_params
692 +
693 +
694 +class Qwen2_5_VLProcessingInfo(Qwen2VLProcessingInfo):
695 +
696 + def get_hf_config(self):
697 + return self.ctx.get_hf_config(Qwen2_5_VLConfig)
698 +
699 + def get_hf_processor(
700 + self,
701 + *,
702 + min_pixels: Optional[int] = None,
703 + max_pixels: Optional[int] = None,
704 + fps: Optional[float] = 2.0,
705 + ) -> Qwen2_5_VLProcessor:
706 + hf_processor = self.ctx.get_hf_processor(Qwen2_5_VLProcessor)
707 + image_processor = hf_processor.image_processor # type: ignore
708 + assert isinstance(image_processor, Qwen2_5_VLImageProcessor)
709 +
710 + if min_pixels:
711 + image_processor.min_pixels = min_pixels
712 + if max_pixels:
713 + image_processor.max_pixels = max_pixels
714 + if max_pixels or min_pixels:
715 + image_processor.size = {
716 + "min_pixels": image_processor.min_pixels,
717 + "max_pixels": image_processor.max_pixels,
718 + }
719 +
720 + return hf_processor
721 +
722 + def get_image_processor(
723 + self,
724 + *,
725 + min_pixels: Optional[int] = None,
726 + max_pixels: Optional[int] = None,
727 + fps: Optional[float] = 2.0,
728 + ) -> Qwen2_5_VLImageProcessor:
729 + hf_processor = self.get_hf_processor(
730 + min_pixels=min_pixels,
731 + max_pixels=max_pixels,
732 + fps=fps,
733 + )
734 + image_processor = hf_processor.image_processor # type: ignore
735 + assert isinstance(image_processor, Qwen2_5_VLImageProcessor)
736 + return image_processor
737 +
738 +
739 +class Qwen2_5_VLMultiModalProcessor(Qwen2VLMultiModalProcessor):
740 +
741 + def _get_mm_fields_config(
742 + self,
743 + hf_inputs: BatchFeature,
744 + hf_processor_mm_kwargs: Mapping[str, object],
745 + ) -> Mapping[str, MultiModalFieldConfig]:
746 + return dict(
747 + **super()._get_mm_fields_config(hf_inputs, hf_processor_mm_kwargs),
748 + second_per_grid_ts=MultiModalFieldConfig.batched("video"),
749 + )
750 +
751 +
752 +@MULTIMODAL_REGISTRY.register_processor(
753 + Qwen2_5_VLMultiModalProcessor,
754 + info=Qwen2_5_VLProcessingInfo,
755 + dummy_inputs=Qwen2_5_VLDummyInputsBuilder)
756 +class Qwen2_5_VLForConditionalGeneration(nn.Module, SupportsMultiModal,
757 + SupportsLoRA, SupportsPP):
758 + packed_modules_mapping = {
759 + "qkv_proj": [
760 + "q_proj",
761 + "k_proj",
762 + "v_proj",
763 + ]
764 + }
765 +
766 + # LoRA specific attributes, TODO: double check
767 + supported_lora_modules = [
768 + "qkv_proj",
769 + "o_proj",
770 + "gate_up_proj",
771 + "down_proj",
772 + "gate_proj"
773 + "up_proj",
774 + # vision tower
775 + "qkv",
776 + "attn.proj", # Distinguish patch_embed.proj
777 + "fc1",
778 + "fc2",
779 + # projector
780 + "mlp.0",
781 + "mlp.2"
782 + ]
783 + embedding_modules = {}
784 + embedding_padding_modules = []
785 +
786 + # To ensure correct weight loading and mapping.
787 + hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={
788 + "lm_head.": "language_model.lm_head.",
789 + "model.": "language_model.model.",
790 + })
791 +
792 + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
793 + super().__init__()
794 + config: Qwen2_5_VLConfig = vllm_config.model_config.hf_config
795 + quant_config = vllm_config.quant_config
796 + multimodal_config = vllm_config.model_config.multimodal_config
797 +
798 + self.config = config
799 + self.multimodal_config = multimodal_config
800 +
801 + self.visual = Qwen2_5_VisionTransformer(
802 + config.vision_config,
803 + norm_eps=getattr(config, "rms_norm_eps", 1e-6),
804 + quant_config=self._maybe_ignore_quant_config(quant_config),
805 + prefix=maybe_prefix(prefix, "visual"),
806 + )
807 +
808 + self.language_model = init_vllm_registered_model(
809 + vllm_config=vllm_config,
810 + prefix=maybe_prefix(prefix, "language_model"),
811 + architectures=["Qwen2ForCausalLM"],
812 + )
813 +
814 + self.make_empty_intermediate_tensors = (
815 + self.language_model.make_empty_intermediate_tensors)
816 +
817 + @cached_property
818 + def sampler(self):
819 + if hasattr(self.language_model, "sampler"):
820 + return self.language_model.sampler
821 +
822 + return get_sampler()
823 +
824 + def _maybe_ignore_quant_config(self, quant_config: QuantizationConfig):
825 + # GPTQ configs do not have a list of ignored modules, however AutoGPTQ
826 + # seems to avoid vision encoder sections for some models.
827 + if isinstance(quant_config, (GPTQConfig, GPTQMarlinConfig)):
828 + return None
829 + return quant_config
830 +
831 + def _validate_and_reshape_mm_tensor(self, mm_input: object,
832 + name: str) -> torch.Tensor:
833 + if not isinstance(mm_input, (torch.Tensor, list)):
834 + raise ValueError(f"Incorrect type of {name}. "
835 + f"Got type: {type(mm_input)}")
836 + if isinstance(mm_input, torch.Tensor):
837 + if mm_input.ndim == 2:
838 + return mm_input
839 + if mm_input.ndim != 3:
840 + raise ValueError(f"{name} should be 2D or batched 3D tensor. "
841 + f"Got ndim: {mm_input.ndim} "
842 + f"(shape={mm_input.shape})")
843 + return torch.concat(list(mm_input))
844 + else:
845 + return torch.concat(mm_input)
846 +
847 + def _parse_and_validate_image_input(
848 + self, **kwargs: object) -> Optional[Qwen2_5_VLImageInputs]:
849 + pixel_values = kwargs.pop("pixel_values", None)
850 + image_embeds = kwargs.pop("image_embeds", None)
851 + image_grid_thw = kwargs.pop("image_grid_thw", None)
852 +
853 + if pixel_values is None and image_embeds is None:
854 + return None
855 +
856 + if pixel_values is not None:
857 + pixel_values = self._validate_and_reshape_mm_tensor(
858 + pixel_values, "image pixel values")
859 + image_grid_thw = self._validate_and_reshape_mm_tensor(
860 + image_grid_thw, "image grid_thw")
861 +
862 + if not isinstance(pixel_values, (torch.Tensor, list)):
863 + raise ValueError("Incorrect type of image pixel values. "
864 + f"Got type: {type(pixel_values)}")
865 +
866 + return Qwen2_5_VLImagePixelInputs(type="pixel_values",
867 + pixel_values=pixel_values,
868 + image_grid_thw=image_grid_thw)
869 +
870 + if image_embeds is not None:
871 + image_embeds = self._validate_and_reshape_mm_tensor(
872 + image_embeds, "image embeds")
873 + image_grid_thw = self._validate_and_reshape_mm_tensor(
874 + image_grid_thw, "image grid_thw")
875 +
876 + if not isinstance(image_embeds, torch.Tensor):
877 + raise ValueError("Incorrect type of image embeddings. "
878 + f"Got type: {type(image_embeds)}")
879 + return Qwen2_5_VLImageEmbeddingInputs(
880 + type="image_embeds",
881 + image_embeds=image_embeds,
882 + image_grid_thw=image_grid_thw)
883 +
884 + def _parse_and_validate_video_input(
885 + self, **kwargs: object) -> Optional[Qwen2_5_VLVideoInputs]:
886 + pixel_values_videos = kwargs.pop("pixel_values_videos", None)
887 + video_embeds = kwargs.pop("video_embeds", None)
888 + video_grid_thw = kwargs.pop("video_grid_thw", None)
889 + second_per_grid_ts = kwargs.pop("second_per_grid_ts", None)
890 +
891 + if pixel_values_videos is None and video_embeds is None:
892 + return None
893 +
894 + if pixel_values_videos is not None:
895 + pixel_values_videos = self._validate_and_reshape_mm_tensor(
896 + pixel_values_videos, "video pixel values")
897 + video_grid_thw = self._validate_and_reshape_mm_tensor(
898 + video_grid_thw, "video grid_thw")
899 +
900 + return Qwen2_5_VLVideoPixelInputs(
901 + type="pixel_values_videos",
902 + pixel_values_videos=pixel_values_videos,
903 + video_grid_thw=video_grid_thw,
904 + second_per_grid_ts=second_per_grid_ts,
905 + )
906 +
907 + if video_embeds is not None:
908 + video_embeds = self._validate_and_reshape_mm_tensor(
909 + video_embeds, "video embeds")
910 + video_grid_thw = self._validate_and_reshape_mm_tensor(
911 + video_grid_thw, "video grid_thw")
912 +
913 + if not isinstance(video_embeds, torch.Tensor):
914 + raise ValueError("Incorrect type of video embeddings. "
915 + f"Got type: {type(video_embeds)}")
916 + return Qwen2_5_VLVideoEmbeddingInputs(
917 + type="video_embeds",
918 + video_embeds=video_embeds,
919 + video_grid_thw=video_grid_thw)
920 +
921 + def _process_image_input(
922 + self,
923 + image_input: Qwen2_5_VLImageInputs) -> tuple[torch.Tensor, ...]:
924 +
925 + grid_thw = image_input["image_grid_thw"]
926 + assert grid_thw.ndim == 2
927 +
928 + if image_input["type"] == "image_embeds":
929 + image_embeds = image_input["image_embeds"].type(self.visual.dtype)
930 + else:
931 + pixel_values = image_input["pixel_values"].type(self.visual.dtype)
932 + image_embeds = self.visual(pixel_values, grid_thw=grid_thw)
933 +
934 + # Split concatenated embeddings for each image item.
935 + merge_size = self.visual.spatial_merge_size
936 + sizes = grid_thw.prod(-1) // merge_size // merge_size
937 +
938 + return image_embeds.split(sizes.tolist())
939 +
940 + def _process_video_input(
941 + self,
942 + video_input: Qwen2_5_VLVideoInputs) -> tuple[torch.Tensor, ...]:
943 +
944 + grid_thw = video_input["video_grid_thw"]
945 + assert grid_thw.ndim == 2
946 +
947 + if video_input["type"] == "video_embeds":
948 + video_embeds = video_input["video_embeds"].type(self.visual.dtype)
949 + else:
950 + pixel_values_videos = video_input["pixel_values_videos"].type(
951 + self.visual.dtype)
952 + video_embeds = self.visual(pixel_values_videos, grid_thw=grid_thw)
953 +
954 + # Split concatenated embeddings for each video item.
955 + merge_size = self.visual.spatial_merge_size
956 + sizes = grid_thw.prod(-1) // merge_size // merge_size
957 +
958 + return video_embeds.split(sizes.tolist())
959 +
960 + def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
961 + modalities = {}
962 +
963 + # Preserve the order of modalities if there are multiple of them
964 + # from the order of kwargs.
965 + for input_key in kwargs:
966 + if input_key in ("pixel_values",
967 + "image_embeds") and "images" not in modalities:
968 + modalities["images"] = self._parse_and_validate_image_input(
969 + **kwargs)
970 + if input_key in ("pixel_values_videos",
971 + "video_embeds") and "videos" not in modalities:
972 + modalities["videos"] = self._parse_and_validate_video_input(
973 + **kwargs)
974 + return modalities
975 +
976 + def get_multimodal_embeddings(
977 + self, **kwargs) -> Optional[tuple[torch.Tensor, ...]]:
978 +
979 + modalities = self._parse_and_validate_multimodal_inputs(**kwargs)
980 + if not modalities:
981 + return None
982 +
983 + # The result multimodal_embeddings is tuple of tensors, with each
984 + # tensor correspoending to a multimodal data item (image or video).
985 + multimodal_embeddings: tuple[torch.Tensor, ...] = ()
986 +
987 + # NOTE: It is important to iterate over the keys in this dictionary
988 + # to preserve the order of the modalities.
989 + for modality in modalities:
990 + if modality == "images":
991 + image_input = modalities["images"]
992 + vision_embeddings = self._process_image_input(image_input)
993 + multimodal_embeddings += vision_embeddings
994 + if modality == "videos":
995 + video_input = modalities["videos"]
996 + video_embeddings = self._process_video_input(video_input)
997 + multimodal_embeddings += video_embeddings
998 + return multimodal_embeddings
999 +
1000 + def get_input_embeddings(
1001 + self,
1002 + input_ids: torch.Tensor,
1003 + multimodal_embeddings: Optional[tuple[torch.Tensor, ...]] = None,
1004 + ) -> torch.Tensor:
1005 + inputs_embeds = self.language_model.get_input_embeddings(input_ids)
1006 + if multimodal_embeddings is not None:
1007 + inputs_embeds = merge_multimodal_embeddings(
1008 + input_ids, inputs_embeds, multimodal_embeddings,
1009 + [self.config.image_token_id, self.config.video_token_id])
1010 + return inputs_embeds
1011 +
1012 + def get_input_embeddings_v0(
1013 + self,
1014 + input_ids: torch.Tensor,
1015 + image_input: Optional[tuple[torch.Tensor, ...]] = None,
1016 + video_input: Optional[tuple[torch.Tensor, ...]] = None,
1017 + ) -> torch.Tensor:
1018 +
1019 + inputs_embeds = self.get_input_embeddings(input_ids)
1020 + if image_input is not None:
1021 + image_embeds = self._process_image_input(image_input)
1022 + inputs_embeds = merge_multimodal_embeddings(
1023 + input_ids,
1024 + inputs_embeds,
1025 + image_embeds,
1026 + placeholder_token_id=self.config.image_token_id,
1027 + )
1028 +
1029 + if video_input is not None:
1030 + video_embeds = self._process_video_input(video_input)
1031 + inputs_embeds = merge_multimodal_embeddings(
1032 + input_ids,
1033 + inputs_embeds,
1034 + video_embeds,
1035 + placeholder_token_id=self.config.video_token_id,
1036 + )
1037 + return inputs_embeds
1038 +
1039 + def forward(
1040 + self,
1041 + input_ids: torch.Tensor,
1042 + positions: torch.Tensor,
1043 + kv_caches: List[torch.Tensor],
1044 + attn_metadata: AttentionMetadata,
1045 + intermediate_tensors: Optional[IntermediateTensors] = None,
1046 + inputs_embeds: Optional[torch.Tensor] = None,
1047 + **kwargs: object,
1048 + ) -> Union[torch.Tensor, IntermediateTensors]:
1049 + """Run forward pass for Qwen2.5-VL.
1050 +
1051 + Args:
1052 + input_ids: Flattened (concatenated) input_ids corresponding to a
1053 + batch.
1054 + positions: Flattened (concatenated) position ids corresponding to a
1055 + batch.
1056 + **NOTE**: If mrope is enabled (default setting for Qwen2.5-VL
1057 + opensource models), the shape will be `(3, seq_len)`,
1058 + otherwise it will be `(seq_len,).
1059 + pixel_values: Pixel values to be fed to a model.
1060 + `None` if no images are passed.
1061 + image_grid_thw: Tensor `(n_images, 3)` of image 3D grid in LLM.
1062 + `None` if no images are passed.
1063 + pixel_values_videos: Pixel values of videos to be fed to a model.
1064 + `None` if no videos are passed.
1065 + video_grid_thw: Tensor `(n_videos, 3)` of video 3D grid in LLM.
1066 + `None` if no videos are passed.
1067 + second_per_grid_ts: Tensor `(num_videos)` of video time interval (
1068 + in seconds) for each grid along the temporal dimension in the
1069 + 3D position IDs. `None` if no videos are passed.
1070 + """
1071 +
1072 + if intermediate_tensors is not None:
1073 + inputs_embeds = None
1074 +
1075 + # NOTE: In v1, inputs_embeds is always generated at model runner from
1076 + # `get_multimodal_embeddings` and `get_input_embeddings`, this
1077 + # condition is only for v0 compatibility.
1078 + elif inputs_embeds is None:
1079 + image_input = self._parse_and_validate_image_input(**kwargs)
1080 + video_input = self._parse_and_validate_video_input(**kwargs)
1081 +
1082 + if image_input is None and video_input is None:
1083 + inputs_embeds = None
1084 + else:
1085 + if uses_mrope(self.config):
1086 + assert positions.ndim == 2 and positions.size(0) == 3, (
1087 + "multimodal section rotary embedding requires "
1088 + f"(3, seq_len) positions, but got {positions.size()}")
1089 + inputs_embeds = self.get_input_embeddings_v0(
1090 + input_ids,
1091 + image_input=image_input,
1092 + video_input=video_input)
1093 + input_ids = None
1094 +
1095 + hidden_states = self.language_model.model(
1096 + input_ids=input_ids,
1097 + positions=positions,
1098 + kv_caches=kv_caches,
1099 + attn_metadata=attn_metadata,
1100 + intermediate_tensors=intermediate_tensors,
1101 + inputs_embeds=inputs_embeds,
1102 + )
1103 + return hidden_states
1104 +
1105 + def compute_logits(
1106 + self,
1107 + hidden_states: torch.Tensor,
1108 + sampling_metadata: SamplingMetadata,
1109 + ) -> Optional[torch.Tensor]:
1110 + return self.language_model.compute_logits(hidden_states,
1111 + sampling_metadata)
1112 +
1113 + def sample(
1114 + self,
1115 + logits: torch.Tensor,
1116 + sampling_metadata: SamplingMetadata,
1117 + ) -> Optional[SamplerOutput]:
1118 + return self.language_model.sample(logits, sampling_metadata)
1119 +
1120 + def load_weights(self, weights: Iterable[Tuple[str,
1121 + torch.Tensor]]) -> Set[str]:
1122 +
1123 + loader = AutoWeightsLoader(self)
1124 + return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
1125 +
1126 + def get_mm_mapping(self) -> MultiModelKeys:
1127 + """
1128 + Get the module prefix in multimodal models
1129 + """
1130 + return MultiModelKeys.from_string_field(
1131 + language_model="language_model",
1132 + connector="visual.",
1133 + tower_model="visual.merger.")
vllm/model_executor/models/qwen2_vl.py
+8 -8
@@ -650,8 +650,8 @@ class Qwen2VisionTransformer(nn.Module):
650 return loaded_params
651
652
653 -class Qwen2EmbeddingItems(ModalityDataItems[dict[str, torch.Tensor],
654 - dict[str, torch.Tensor]]):
653 +class Qwen2VLEmbeddingItems(ModalityDataItems[dict[str, torch.Tensor],
654 + dict[str, torch.Tensor]]):
655
656 def __init__(self, data: dict, modality: str) -> None:
657 super().__init__(data, modality)
@@ -683,26 +683,26 @@ class Qwen2EmbeddingItems(ModalityDataItems[dict[str, torch.Tensor],
683 return self.data
684
685
686 -class Qwen2ImageEmbeddingItems(Qwen2EmbeddingItems):
686 +class Qwen2VLImageEmbeddingItems(Qwen2VLEmbeddingItems):
687
688 def __init__(self, data: dict) -> None:
689 super().__init__(data, "image")
690
691
692 -class Qwen2VideoEmbeddingItems(Qwen2EmbeddingItems):
692 +class Qwen2VLVideoEmbeddingItems(Qwen2VLEmbeddingItems):
693
694 def __init__(self, data: dict) -> None:
695 super().__init__(data, "video")
696
697
698 -class Qwen2MultiModalDataParser(MultiModalDataParser):
698 +class Qwen2VLMultiModalDataParser(MultiModalDataParser):
699
700 def _parse_image_data(
701 self,
702 data: Union[dict[str, torch.Tensor], ModalityData[ImageItem]],
703 ) -> ModalityDataItems[Any, Any]:
704 if isinstance(data, dict):
705 - return Qwen2EmbeddingItems(data, modality="image")
705 + return Qwen2VLEmbeddingItems(data, modality="image")
706
707 return super()._parse_image_data(data)
708
@@ -711,7 +711,7 @@ class Qwen2MultiModalDataParser(MultiModalDataParser):
711 data: Union[dict[str, torch.Tensor], ModalityData[VideoItem]],
712 ) -> ModalityDataItems[Any, Any]:
713 if isinstance(data, dict):
714 - return Qwen2EmbeddingItems(data, modality="video")
714 + return Qwen2VLEmbeddingItems(data, modality="video")
715
716 return super()._parse_video_data(data)
717
@@ -948,7 +948,7 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor[Qwen2VLProcessingInfo]
948 ):
949
950 def _get_data_parser(self) -> MultiModalDataParser:
951 - return Qwen2MultiModalDataParser()
951 + return Qwen2VLMultiModalDataParser()
952
953 def _get_prompt_replacements(
954 self,
vllm/model_executor/models/registry.py
+1
@@ -172,6 +172,7 @@ _MULTIMODAL_MODELS = {
172 "PixtralForConditionalGeneration": ("pixtral", "PixtralForConditionalGeneration"), # noqa: E501
173 "QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
174 "Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"), # noqa: E501
175 + "Qwen2_5_VLForConditionalGeneration": ("qwen2_5_vl", "Qwen2_5_VLForConditionalGeneration"), # noqa: E501
176 "Qwen2AudioForConditionalGeneration": ("qwen2_audio", "Qwen2AudioForConditionalGeneration"), # noqa: E501
177 "UltravoxModel": ("ultravox", "UltravoxModel"),
178 # [Encoder-decoder]
vllm/v1/worker/gpu_model_runner.py
+6 -6
@@ -285,6 +285,7 @@ class GPUModelRunner:
285 if self.model_config.uses_mrope:
286 image_grid_thw = []
287 video_grid_thw = []
288 + second_per_grid_ts = []
289 for mm_input in self.requests[req_id].mm_inputs:
290 if mm_input.get("image_grid_thw") is not None:
291 image_grid_thw.extend(
@@ -292,6 +293,9 @@ class GPUModelRunner:
293 if mm_input.get("video_grid_thw") is not None:
294 video_grid_thw.extend(
295 mm_input["video_grid_thw"].tolist())
296 + if mm_input.get("second_per_grid_ts") is not None:
297 + second_per_grid_ts.extend(
298 + mm_input["second_per_grid_ts"])
299
300 hf_config = self.model_config.hf_config
301
@@ -299,14 +303,10 @@ class GPUModelRunner:
303 self.requests[req_id].mrope_position_delta = \
304 MRotaryEmbedding.get_input_positions_tensor(
305 self.requests[req_id].prompt_token_ids,
306 + hf_config=hf_config,
307 image_grid_thw=image_grid_thw,
308 video_grid_thw=video_grid_thw,
304 - image_token_id=hf_config.image_token_id,
305 - video_token_id=hf_config.video_token_id,
306 - vision_start_token_id=hf_config.vision_start_token_id,
307 - vision_end_token_id=hf_config.vision_end_token_id,
308 - spatial_merge_size=hf_config.vision_config.
309 - spatial_merge_size,
309 + second_per_grid_ts=second_per_grid_ts,
310 )
311
312 req_ids_to_add.append(req_id)
vllm/worker/cpu_model_runner.py
+3 -6
@@ -386,20 +386,17 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
386 "mrope embedding type requires multi-modal input mapper "
387 "returns 'image_grid_thw' or 'video_grid_thw'.")
388
389 + second_per_grid_ts = mm_kwargs.get("second_per_grid_ts", None)
390 hf_config = self.runner.model_config.hf_config
391 token_ids = seq_data.get_token_ids()
392
393 mrope_positions, mrope_position_delta = \
394 MRotaryEmbedding.get_input_positions(
395 token_ids,
396 + hf_config=hf_config,
397 image_grid_thw=image_grid_thw,
398 video_grid_thw=video_grid_thw,
397 - image_token_id=hf_config.image_token_id,
398 - video_token_id=hf_config.video_token_id,
399 - vision_start_token_id=hf_config.vision_start_token_id,
400 - vision_end_token_id=hf_config.vision_end_token_id,
401 - spatial_merge_size=hf_config.vision_config.
402 - spatial_merge_size,
399 + second_per_grid_ts=second_per_grid_ts,
400 context_len=computed_len,
401 )
402 seq_data.mrope_position_delta = mrope_position_delta
vllm/worker/model_runner.py
+3 -6
@@ -702,6 +702,7 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
702 "mrope embedding type requires multi-modal input mapper "
703 "returns 'image_grid_thw' or 'video_grid_thw'.")
704
705 + second_per_grid_ts = mm_kwargs.get("second_per_grid_ts", None)
706 hf_config = self.runner.model_config.hf_config
707
708 inter_data.mrope_input_positions = [None] * inter_data.n_seqs
@@ -713,14 +714,10 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
714 mrope_input_positions, mrope_position_delta = \
715 MRotaryEmbedding.get_input_positions(
716 token_ids,
717 + hf_config=hf_config,
718 image_grid_thw=image_grid_thw,
719 video_grid_thw=video_grid_thw,
718 - image_token_id=hf_config.image_token_id,
719 - video_token_id=hf_config.video_token_id,
720 - vision_start_token_id=hf_config.vision_start_token_id,
721 - vision_end_token_id=hf_config.vision_end_token_id,
722 - spatial_merge_size=hf_config.vision_config.
723 - spatial_merge_size,
720 + second_per_grid_ts=second_per_grid_ts,
721 context_len=inter_data.context_lens[seq_idx],
722 seq_len=inter_data.seq_lens[seq_idx],
723 )