A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 00:45 UTC
[VLM] Qwen2.5-VL
Roger Wang committed
Feb 5, 2025 at 13:31 UTC
bf3b79efb82676219a3275764d8fcf4c70097ce5
14 files changed
+1315
-52
docs/source/models/supported_models.md
+11
@@ -846,6 +846,13 @@ See [this page](#generative-models) for more information on how to use generativ
846
* ✅︎
847
* ✅︎
848
* ✅︎
849
+- * `Qwen2_5_VLForConditionalGeneration`
850
+ * Qwen2.5-VL
851
+ * T + I<sup>E+</sup> + V<sup>E+</sup>
852
+ * `Qwen/Qwen2.5-VL-3B-Instruct`, `Qwen/Qwen2.5-VL-72B-Instruct`, etc.
853
+ *
854
+ * ✅︎
855
+ * ✅︎
856
- * `UltravoxModel`
857
* Ultravox
858
* T + A<sup>E+</sup>
@@ -880,6 +887,10 @@ The chat template for Pixtral-HF is incorrect (see [discussion](https://huggingf
887
A corrected version is available at <gh-file:examples/template_pixtral_hf.jinja>.
888
:::
889
890
+:::{note}
891
+To use Qwen2.5-VL series models, you have to install Huggingface `transformers` library from source via `pip install git+https://github.com/huggingface/transformers`.
892
+:::
893
+
894
### Pooling Models
895
896
See [this page](pooling-models) for more information on how to use pooling models.
examples/offline_inference/vision_language.py
+31
@@ -531,6 +531,36 @@ def run_qwen2_vl(question: str, modality: str):
531
return llm, prompt, stop_token_ids
532
533
534
+# Qwen2.5-VL
535
+def run_qwen2_5_vl(question: str, modality: str):
536
+
537
+ model_name = "Qwen/Qwen2.5-VL-3B-Instruct"
538
+
539
+ llm = LLM(
540
+ model=model_name,
541
+ max_model_len=4096,
542
+ max_num_seqs=5,
543
+ mm_processor_kwargs={
544
+ "min_pixels": 28 * 28,
545
+ "max_pixels": 1280 * 28 * 28,
546
+ "fps": 1,
547
+ },
548
+ disable_mm_preprocessor_cache=args.disable_mm_preprocessor_cache,
549
+ )
550
+
551
+ if modality == "image":
552
+ placeholder = "<|image_pad|>"
553
+ elif modality == "video":
554
+ placeholder = "<|video_pad|>"
555
+
556
+ prompt = ("<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
557
+ f"<|im_start|>user\n<|vision_start|>{placeholder}<|vision_end|>"
558
+ f"{question}<|im_end|>\n"
559
+ "<|im_start|>assistant\n")
560
+ stop_token_ids = None
561
+ return llm, prompt, stop_token_ids
562
+
563
+
564
model_example_map = {
565
"aria": run_aria,
566
"blip-2": run_blip2,
@@ -557,6 +587,7 @@ model_example_map = {
587
"pixtral_hf": run_pixtral_hf,
588
"qwen_vl": run_qwen_vl,
589
"qwen2_vl": run_qwen2_vl,
590
+ "qwen2_5_vl": run_qwen2_5_vl,
591
}
592
593
examples/offline_inference/vision_language_multi_image.py
+58
@@ -392,6 +392,63 @@ def load_qwen2_vl(question, image_urls: List[str]) -> ModelRequestData:
392
)
393
394
395
+def load_qwen2_5_vl(question, image_urls: List[str]) -> ModelRequestData:
396
+ try:
397
+ from qwen_vl_utils import process_vision_info
398
+ except ModuleNotFoundError:
399
+ print('WARNING: `qwen-vl-utils` not installed, input images will not '
400
+ 'be automatically resized. You can enable this functionality by '
401
+ '`pip install qwen-vl-utils`.')
402
+ process_vision_info = None
403
+
404
+ model_name = "Qwen/Qwen2.5-VL-3B-Instruct"
405
+
406
+ llm = LLM(
407
+ model=model_name,
408
+ max_model_len=32768 if process_vision_info is None else 4096,
409
+ max_num_seqs=5,
410
+ limit_mm_per_prompt={"image": len(image_urls)},
411
+ )
412
+
413
+ placeholders = [{"type": "image", "image": url} for url in image_urls]
414
+ messages = [{
415
+ "role": "system",
416
+ "content": "You are a helpful assistant."
417
+ }, {
418
+ "role":
419
+ "user",
420
+ "content": [
421
+ *placeholders,
422
+ {
423
+ "type": "text",
424
+ "text": question
425
+ },
426
+ ],
427
+ }]
428
+
429
+ processor = AutoProcessor.from_pretrained(model_name)
430
+
431
+ prompt = processor.apply_chat_template(messages,
432
+ tokenize=False,
433
+ add_generation_prompt=True)
434
+
435
+ stop_token_ids = None
436
+
437
+ if process_vision_info is None:
438
+ image_data = [fetch_image(url) for url in image_urls]
439
+ else:
440
+ image_data, _ = process_vision_info(messages,
441
+ return_video_sample_fps=False)
442
+
443
+ return ModelRequestData(
444
+ llm=llm,
445
+ prompt=prompt,
446
+ stop_token_ids=stop_token_ids,
447
+ image_data=image_data,
448
+ chat_template=None,
449
+ )
450
+
451
+
452
model_example_map = {
453
"aria": load_aria,
454
"deepseek_vl_v2": load_deepseek_vl2,
@@ -404,6 +461,7 @@ model_example_map = {
461
"pixtral_hf": load_pixtral_hf,
462
"qwen_vl_chat": load_qwen_vl_chat,
463
"qwen2_vl": load_qwen2_vl,
464
+ "qwen2_5_vl": load_qwen2_5_vl,
465
}
466
467
tests/models/decoder_only/vision_language/test_models.py
+22
@@ -121,6 +121,8 @@ VLM_TEST_SETTINGS = {
121
else ("half", "float")),
122
marks=[pytest.mark.core_model],
123
),
124
+ # TODO(ywang96): Move Qwen2-VL out of core models in favor of Qwen2.5-VL
125
+ # once we upgraded to transformers>=4.49.0.
126
"qwen2_vl": VLMTestInfo(
127
models=["Qwen/Qwen2-VL-2B-Instruct"],
128
test_type=(
@@ -138,6 +140,26 @@ VLM_TEST_SETTINGS = {
140
image_size_factors=[(), (0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
141
marks=[pytest.mark.core_model, pytest.mark.cpu_model],
142
),
143
+ "qwen2_5_vl": VLMTestInfo(
144
+ models=["Qwen/Qwen2.5-VL-3B-Instruct"],
145
+ test_type=(
146
+ VLMTestType.IMAGE,
147
+ VLMTestType.MULTI_IMAGE,
148
+ VLMTestType.VIDEO
149
+ ),
150
+ prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
151
+ img_idx_to_prompt=lambda idx: "<|vision_start|><|image_pad|><|vision_end|>", # noqa: E501
152
+ video_idx_to_prompt=lambda idx: "<|vision_start|><|video_pad|><|vision_end|>", # noqa: E501
153
+ max_model_len=4096,
154
+ max_num_seqs=2,
155
+ auto_cls=AutoModelForVision2Seq,
156
+ vllm_output_post_proc=model_utils.qwen2_vllm_to_hf_output,
157
+ image_size_factors=[(), (0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
158
+ marks=[pytest.mark.skipif(
159
+ TRANSFORMERS_VERSION < "4.49.0",
160
+ reason="HF model requires transformers>=4.49.0",
161
+ ), pytest.mark.core_model, pytest.mark.cpu_model],
162
+ ),
163
#### Extended model tests
164
"aria": VLMTestInfo(
165
models=["rhymes-ai/Aria"],
tests/models/multimodal/processing/test_common.py
+1
@@ -161,6 +161,7 @@ def _test_processing_correctness(
161
"nvidia/NVLM-D-72B",
162
"Qwen/Qwen-VL-Chat",
163
"Qwen/Qwen2-VL-2B-Instruct",
164
+ "Qwen/Qwen2.5-VL-3B-Instruct",
165
"Qwen/Qwen2-Audio-7B-Instruct",
166
"fixie-ai/ultravox-v0_3",
167
])
tests/models/registry.py
+2
@@ -264,6 +264,8 @@ _MULTIMODAL_EXAMPLE_MODELS = {
264
trust_remote_code=True),
265
"Qwen2AudioForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2-Audio-7B-Instruct"), # noqa: E501
266
"Qwen2VLForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2-VL-2B-Instruct"), # noqa: E501
267
+ "Qwen2_5_VLForConditionalGeneration": _HfExamplesInfo("Qwen/Qwen2.5-VL-3B-Instruct", # noqa: E501
268
+ min_transformers_version="4.49"), # noqa: E501
269
"UltravoxModel": _HfExamplesInfo("fixie-ai/ultravox-v0_3",
270
trust_remote_code=True),
271
# [Encoder-decoder]
vllm/entrypoints/chat_utils.py
+2
-2
@@ -410,7 +410,7 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
410
return "<image>"
411
if model_type == "mllama":
412
return "<|image|>"
413
- if model_type == "qwen2_vl":
413
+ if model_type in ("qwen2_vl", "qwen2_5_vl"):
414
return "<|vision_start|><|image_pad|><|vision_end|>"
415
if model_type == "molmo":
416
return ""
@@ -430,7 +430,7 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
430
return "(<audio>./</audio>)"
431
raise TypeError(f"Unknown model type: {model_type}")
432
elif modality == "video":
433
- if model_type == "qwen2_vl":
433
+ if model_type in ("qwen2_vl", "qwen2_5_vl"):
434
return "<|vision_start|><|video_pad|><|vision_end|>"
435
if model_type in ("minicpmo", "minicpmv"):
436
return "(<video>./</video>)"
vllm/model_executor/layers/rotary_embedding.py
+34
-24
@@ -27,6 +27,7 @@ from typing import Any, Dict, List, Optional, Tuple, Union
27
28
import torch
29
import torch.nn as nn
30
+from transformers import PretrainedConfig
31
32
from vllm.model_executor.custom_op import CustomOp
33
@@ -772,8 +773,12 @@ class MRotaryEmbedding(RotaryEmbedding):
773
dtype: torch.dtype,
774
mrope_section: Optional[List[int]] = None,
775
) -> None:
775
- super().__init__(head_size, rotary_dim, max_position_embeddings, base,
776
- is_neox_style, dtype)
776
+ # In Qwen2.5-VL, the maximum index value is related to the duration of
777
+ # the input video. We enlarge max_position_embeddings to 4 times to get
778
+ # a larger the cos and sin cache.
779
+ self.cache_max_position_num = max_position_embeddings * 4
780
+ super().__init__(head_size, rotary_dim, self.cache_max_position_num,
781
+ base, is_neox_style, dtype)
782
783
self.mrope_section = mrope_section
784
if self.mrope_section:
@@ -831,13 +836,10 @@ class MRotaryEmbedding(RotaryEmbedding):
836
@staticmethod
837
def get_input_positions(
838
input_tokens: List[int],
839
+ hf_config: PretrainedConfig,
840
image_grid_thw: Union[List[List[int]], torch.Tensor],
841
video_grid_thw: Union[List[List[int]], torch.Tensor],
836
- image_token_id: int,
837
- video_token_id: int,
838
- vision_start_token_id: int,
839
- vision_end_token_id: int,
840
- spatial_merge_size: int,
842
+ second_per_grid_ts: Optional[List[float]] = None,
843
context_len: int = 0,
844
seq_len: Optional[int] = None,
845
) -> Tuple[List[List[int]], int]:
@@ -845,16 +847,13 @@ class MRotaryEmbedding(RotaryEmbedding):
847
848
llm_positions, mrope_position_delta = \
849
MRotaryEmbedding.get_input_positions_tensor(
848
- input_tokens,
849
- image_grid_thw,
850
- video_grid_thw,
851
- image_token_id,
852
- video_token_id,
853
- vision_start_token_id,
854
- vision_end_token_id,
855
- spatial_merge_size,
856
- context_len,
857
- seq_len,
850
+ input_tokens=input_tokens,
851
+ hf_config=hf_config,
852
+ image_grid_thw=image_grid_thw,
853
+ video_grid_thw=video_grid_thw,
854
+ second_per_grid_ts=second_per_grid_ts,
855
+ context_len=context_len,
856
+ seq_len=seq_len,
857
)
858
859
return llm_positions.tolist(), mrope_position_delta
@@ -862,18 +861,22 @@ class MRotaryEmbedding(RotaryEmbedding):
861
@staticmethod
862
def get_input_positions_tensor(
863
input_tokens: List[int],
864
+ hf_config: PretrainedConfig,
865
image_grid_thw: Union[List[List[int]], torch.Tensor],
866
video_grid_thw: Union[List[List[int]], torch.Tensor],
867
- image_token_id: int,
868
- video_token_id: int,
869
- vision_start_token_id: int,
870
- vision_end_token_id: int,
871
- spatial_merge_size: int,
867
+ second_per_grid_ts: Optional[List[float]] = None,
868
context_len: int = 0,
869
seq_len: Optional[int] = None,
870
) -> Tuple[torch.Tensor, int]:
871
"""Get mrope input positions and delta value."""
872
873
+ image_token_id = hf_config.image_token_id
874
+ video_token_id = hf_config.video_token_id
875
+ vision_start_token_id = hf_config.vision_start_token_id
876
+ spatial_merge_size = hf_config.vision_config.spatial_merge_size
877
+ tokens_per_second = getattr(hf_config.vision_config,
878
+ "tokens_per_second", 1.0)
879
+
880
if isinstance(image_grid_thw, torch.Tensor):
881
image_grid_thw = image_grid_thw.tolist()
882
if isinstance(video_grid_thw, torch.Tensor):
@@ -892,6 +895,7 @@ class MRotaryEmbedding(RotaryEmbedding):
895
896
image_index, video_index = 0, 0
897
for _ in range(image_nums + video_nums):
898
+ video_second_per_grid_t = 0.0
899
if image_token_id in input_tokens and remain_images > 0:
900
ed_image = input_tokens.index(image_token_id, st)
901
else:
@@ -915,9 +919,13 @@ class MRotaryEmbedding(RotaryEmbedding):
919
video_grid_thw[video_index][1],
920
video_grid_thw[video_index][2],
921
)
922
+ video_second_per_grid_t = 1.0
923
+ if second_per_grid_ts is not None:
924
+ video_second_per_grid_t = second_per_grid_ts[video_index]
925
video_index += 1
926
remain_videos -= 1
927
ed = ed_video
928
+
929
llm_grid_t, llm_grid_h, llm_grid_w = \
930
t, h // spatial_merge_size, w // spatial_merge_size
931
text_len = ed - st
@@ -927,8 +935,10 @@ class MRotaryEmbedding(RotaryEmbedding):
935
llm_pos_ids_list.append(
936
torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx)
937
930
- t_index = torch.arange(llm_grid_t).view(-1, 1).expand(
931
- -1, llm_grid_h * llm_grid_w).flatten()
938
+ t_index = (torch.arange(llm_grid_t).view(-1, 1).expand(
939
+ -1, llm_grid_h * llm_grid_w) * video_second_per_grid_t *
940
+ tokens_per_second).long().flatten()
941
+
942
h_index = torch.arange(llm_grid_h).view(1, -1, 1).expand(
943
llm_grid_t, -1, llm_grid_w).flatten()
944
w_index = torch.arange(llm_grid_w).view(1, 1, -1).expand(
vllm/model_executor/models/qwen2_5_vl.py
new
+1133
@@ -0,0 +1,1133 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+
3
+# Adapted from
4
+# https://github.com/huggingface/transformers/blob/main/src/transformers/models/qwen2_5_vl/modeling_qwen2_5_vl.py
5
+# Copyright 2025 The vLLM team.
6
+# Copyright 2025 The Qwen Team.
7
+# Copyright 2025 The HuggingFace Inc. team.
8
+# All rights reserved.
9
+#
10
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
11
+# and OPT implementations in this library. It has been modified from its
12
+# original forms to accommodate minor architectural differences compared
13
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
14
+#
15
+# Licensed under the Apache License, Version 2.0 (the "License");
16
+# you may not use this file except in compliance with the License.
17
+# You may obtain a copy of the License at
18
+#
19
+# http://www.apache.org/licenses/LICENSE-2.0
20
+#
21
+# Unless required by applicable law or agreed to in writing, software
22
+# distributed under the License is distributed on an "AS IS" BASIS,
23
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
24
+# See the License for the specific language governing permissions and
25
+# limitations under the License.
26
+"""Inference-only Qwen2.5-VL model compatible with HuggingFace weights."""
27
+from functools import cached_property, partial
28
+from typing import (Callable, Iterable, List, Literal, Mapping, Optional, Set,
29
+ Tuple, TypedDict, Union)
30
+
31
+import torch
32
+import torch.nn as nn
33
+import torch.nn.functional as F
34
+from einops import rearrange
35
+from transformers import BatchFeature
36
+from transformers.models.qwen2_5_vl import (Qwen2_5_VLImageProcessor,
37
+ Qwen2_5_VLProcessor)
38
+from transformers.models.qwen2_5_vl.configuration_qwen2_5_vl import (
39
+ Qwen2_5_VLConfig, Qwen2_5_VLVisionConfig)
40
+
41
+from vllm.attention import AttentionMetadata
42
+from vllm.config import VllmConfig
43
+from vllm.distributed import parallel_state
44
+from vllm.distributed import utils as dist_utils
45
+from vllm.logger import init_logger
46
+from vllm.model_executor import SamplingMetadata
47
+from vllm.model_executor.layers.activation import _ACTIVATION_REGISTRY
48
+from vllm.model_executor.layers.linear import (ColumnParallelLinear,
49
+ RowParallelLinear)
50
+from vllm.model_executor.layers.quantization import QuantizationConfig
51
+from vllm.model_executor.layers.quantization.gptq import GPTQConfig
52
+from vllm.model_executor.layers.quantization.gptq_marlin import (
53
+ GPTQMarlinConfig)
54
+from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
55
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
56
+from vllm.model_executor.models.module_mapping import MultiModelKeys
57
+from vllm.multimodal import MULTIMODAL_REGISTRY
58
+from vllm.multimodal.inputs import MultiModalFieldConfig
59
+from vllm.platforms import _Backend
60
+from vllm.sequence import IntermediateTensors
61
+from vllm.transformers_utils.config import uses_mrope
62
+
63
+from .interfaces import SupportsLoRA, SupportsMultiModal, SupportsPP
64
+from .qwen2_vl import Qwen2VLDummyInputsBuilder as Qwen2_5_VLDummyInputsBuilder
65
+from .qwen2_vl import (Qwen2VLMultiModalProcessor, Qwen2VLProcessingInfo,
66
+ apply_rotary_pos_emb_vision)
67
+from .utils import (AutoWeightsLoader, WeightsMapper,
68
+ init_vllm_registered_model, maybe_prefix,
69
+ merge_multimodal_embeddings)
70
+from .vision import get_vit_attn_backend
71
+
72
+logger = init_logger(__name__)
73
+
74
+# === Vision Inputs === #
75
+
76
+
77
+class Qwen2_5_VLImagePixelInputs(TypedDict):
78
+ type: Literal["pixel_values"]
79
+ pixel_values: torch.Tensor
80
+ """Shape:
81
+ `(num_patches, num_channels * patch_size * patch_size)`
82
+ """
83
+
84
+ image_grid_thw: torch.Tensor
85
+ """Shape: `(num_images, 3)`
86
+ This should be in `(grid_t, grid_h, grid_w)` format.
87
+ """
88
+
89
+
90
+class Qwen2_5_VLImageEmbeddingInputs(TypedDict):
91
+ type: Literal["image_embeds"]
92
+ image_embeds: torch.Tensor
93
+ """Supported types:
94
+ - List[`torch.Tensor`]: A list of tensors holding all images' features.
95
+ Each tensor holds an image's features.
96
+ - `torch.Tensor`: A tensor holding all images' features
97
+ (concatenation of all images' feature tensors).
98
+
99
+ Tensor shape: `(num_image_features, hidden_size)`
100
+ - `num_image_features` varies based on
101
+ the number and resolution of the images.
102
+ - `hidden_size` must match the hidden size of language model backbone.
103
+ """
104
+
105
+ image_grid_thw: torch.Tensor
106
+ """Shape: `(num_images, 3)`
107
+ This should be in `(grid_t, grid_h, grid_w)` format.
108
+ """
109
+
110
+
111
+Qwen2_5_VLImageInputs = Union[Qwen2_5_VLImagePixelInputs,
112
+ Qwen2_5_VLImageEmbeddingInputs]
113
+
114
+
115
+class Qwen2_5_VLVideoPixelInputs(TypedDict):
116
+ type: Literal["pixel_values_videos"]
117
+ pixel_values_videos: torch.Tensor
118
+ """Shape:
119
+ `(num_patches,
120
+ num_channels * temporal_patch_size * patch_size * patch_size)`
121
+ """
122
+
123
+ video_grid_thw: torch.Tensor
124
+ """Shape: `(num_videos, 3)`
125
+
126
+ This should be in `(grid_t, grid_h, grid_w)` format.
127
+ """
128
+
129
+ second_per_grid_ts: torch.Tensor
130
+ """
131
+ The video time interval (in seconds) for each grid along the temporal
132
+ dimension in the 3D position IDs. Returned when `videos` is not `None`.
133
+ """
134
+
135
+
136
+class Qwen2_5_VLVideoEmbeddingInputs(TypedDict):
137
+ type: Literal["video_embeds"]
138
+ video_embeds: torch.Tensor
139
+ """Supported types:
140
+ - List[`torch.Tensor`]: A list of tensors holding all videos' features.
141
+ Each tensor holds an video's features.
142
+ - `torch.Tensor`: A tensor holding all videos' features
143
+ (concatenation of all videos' feature tensors).
144
+
145
+ Tensor shape: `(num_image_features, hidden_size)`
146
+ - `num_image_features` varies based on
147
+ the number and resolution of the videos.
148
+ - `hidden_size` must match the hidden size of language model backbone.
149
+ """
150
+
151
+ video_grid_thw: torch.Tensor
152
+ """Shape: `(num_videos, 3)`
153
+ This should be in `(grid_t, grid_h, grid_w)` format.
154
+ """
155
+
156
+
157
+Qwen2_5_VLVideoInputs = Union[Qwen2_5_VLVideoPixelInputs,
158
+ Qwen2_5_VLVideoEmbeddingInputs]
159
+
160
+# === Vision Encoder === #
161
+
162
+
163
+class Qwen2_5_VisionMLP(nn.Module):
164
+
165
+ def __init__(self,
166
+ in_features: int,
167
+ hidden_features: int,
168
+ bias: bool = False,
169
+ act_fn: Callable[[torch.Tensor], torch.Tensor] = F.silu,
170
+ quant_config: Optional[QuantizationConfig] = None,
171
+ prefix: str = ""):
172
+ super().__init__()
173
+ self.gate_proj = ColumnParallelLinear(in_features,
174
+ hidden_features,
175
+ bias=bias,
176
+ quant_config=quant_config,
177
+ prefix=f"{prefix}.gate_proj")
178
+ self.up_proj = ColumnParallelLinear(in_features,
179
+ hidden_features,
180
+ bias=bias,
181
+ quant_config=quant_config,
182
+ prefix=f"{prefix}.up_proj")
183
+ self.down_proj = RowParallelLinear(hidden_features,
184
+ in_features,
185
+ bias=bias,
186
+ quant_config=quant_config,
187
+ prefix=f"{prefix}.down_proj")
188
+ self.act_fn = act_fn
189
+
190
+ def forward(self, x: torch.Tensor):
191
+ x_gate, _ = self.gate_proj(x)
192
+ x_gate = self.act_fn(x_gate)
193
+ x_up, _ = self.up_proj(x)
194
+ x_down, _ = self.down_proj(x_gate * x_up)
195
+ return x_down
196
+
197
+
198
+class Qwen2_5_VisionAttention(nn.Module):
199
+
200
+ def __init__(
201
+ self,
202
+ embed_dim: int,
203
+ num_heads: int,
204
+ projection_size: int,
205
+ quant_config: Optional[QuantizationConfig] = None,
206
+ prefix: str = "",
207
+ ) -> None:
208
+ super().__init__()
209
+ # Per attention head and per partition values.
210
+ world_size = parallel_state.get_tensor_model_parallel_world_size()
211
+ self.hidden_size_per_attention_head = dist_utils.divide(
212
+ projection_size, num_heads)
213
+ self.num_attention_heads_per_partition = dist_utils.divide(
214
+ num_heads, world_size)
215
+
216
+ self.qkv = ColumnParallelLinear(input_size=embed_dim,
217
+ output_size=3 * projection_size,
218
+ quant_config=quant_config,
219
+ prefix=f"{prefix}.qkv")
220
+ self.proj = RowParallelLinear(input_size=projection_size,
221
+ output_size=embed_dim,
222
+ quant_config=quant_config,
223
+ prefix=f"{prefix}.proj")
224
+
225
+ # Detect attention implementation.
226
+ self.attn_backend: _Backend = get_vit_attn_backend(support_fa=True)
227
+ if self.attn_backend not in {
228
+ _Backend.FLASH_ATTN, _Backend.TORCH_SDPA, _Backend.XFORMERS
229
+ }:
230
+ raise RuntimeError(
231
+ f"Qwen2.5-VL does not support {self.attn_backend} backend now."
232
+ )
233
+
234
+ def forward(
235
+ self,
236
+ x: torch.Tensor,
237
+ cu_seqlens: torch.Tensor,
238
+ rotary_pos_emb: torch.Tensor,
239
+ ) -> torch.Tensor:
240
+ # [s, b, c] --> [s, b, head * 3 * head_dim]
241
+ x, _ = self.qkv(x)
242
+
243
+ # [s, b, head * 3 * head_dim] --> [s, b, head, 3 * head_dim]
244
+ new_x_shape = x.size()[:-1] + (
245
+ self.num_attention_heads_per_partition,
246
+ 3 * self.hidden_size_per_attention_head,
247
+ )
248
+ x = x.view(*new_x_shape)
249
+
250
+ # [s, b, head, 3 * head_dim] --> 3 [s, b, head, head_dim]
251
+ q, k, v = dist_utils.split_tensor_along_last_dim(x, 3)
252
+ batch_size = q.shape[1]
253
+
254
+ q, k, v = (rearrange(x, "s b ... -> b s ...").contiguous()
255
+ for x in (q, k, v))
256
+ if rotary_pos_emb is not None:
257
+ q = apply_rotary_pos_emb_vision(q, rotary_pos_emb)
258
+ k = apply_rotary_pos_emb_vision(k, rotary_pos_emb)
259
+
260
+ if self.attn_backend == _Backend.FLASH_ATTN:
261
+ # from vllm_flash_attn.flash_attn_interface import (
262
+ # flash_attn_varlen_func)
263
+ from flash_attn import flash_attn_varlen_func
264
+
265
+ q, k, v = (rearrange(x, "b s ... -> (b s) ...") for x in [q, k, v])
266
+
267
+ max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max().item()
268
+ output = flash_attn_varlen_func(q,
269
+ k,
270
+ v,
271
+ cu_seqlens_q=cu_seqlens,
272
+ cu_seqlens_k=cu_seqlens,
273
+ max_seqlen_q=max_seqlen,
274
+ max_seqlen_k=max_seqlen,
275
+ dropout_p=0,
276
+ causal=False)
277
+
278
+ context_layer = rearrange(output,
279
+ "(b s) ... -> b s ...",
280
+ b=batch_size)
281
+ elif self.attn_backend == _Backend.TORCH_SDPA:
282
+ seq_length = q.size(1)
283
+ q, k, v = (rearrange(x, "b s h d -> b h s d") for x in [q, k, v])
284
+ attention_mask = torch.zeros([1, seq_length, seq_length],
285
+ device=q.device,
286
+ dtype=torch.bool)
287
+ for i in range(1, len(cu_seqlens)):
288
+ attention_mask[..., cu_seqlens[i - 1]:cu_seqlens[i],
289
+ cu_seqlens[i - 1]:cu_seqlens[i]] = True
290
+ output = F.scaled_dot_product_attention(q,
291
+ k,
292
+ v,
293
+ attention_mask,
294
+ dropout_p=0.0)
295
+ context_layer = rearrange(output, "b h s d -> b s h d ")
296
+ elif self.attn_backend == _Backend.XFORMERS:
297
+ from xformers import ops as xops
298
+ from xformers.ops.fmha.attn_bias import BlockDiagonalMask
299
+
300
+ seqlens = (cu_seqlens[1:] - cu_seqlens[:-1]).tolist()
301
+ attn_bias = BlockDiagonalMask.from_seqlens(q_seqlen=seqlens,
302
+ kv_seqlen=None)
303
+
304
+ context_layer = xops.memory_efficient_attention_forward(
305
+ q, k, v, attn_bias=attn_bias, p=0, scale=None)
306
+ context_layer = rearrange(context_layer,
307
+ "b s h d -> s b (h d)").contiguous()
308
+
309
+ output, _ = self.proj(context_layer)
310
+ return output
311
+
312
+
313
+class Qwen2RMSNorm(nn.Module):
314
+
315
+ def __init__(self, hidden_size, eps=1e-6):
316
+ super().__init__()
317
+ self.weight = nn.Parameter(torch.ones(hidden_size))
318
+ self.variance_epsilon = eps
319
+
320
+ def forward(self, hidden_states):
321
+ input_dtype = hidden_states.dtype
322
+ hidden_states = hidden_states.to(torch.float32)
323
+ variance = hidden_states.pow(2).mean(-1, keepdim=True)
324
+ hidden_states = hidden_states * torch.rsqrt(variance +
325
+ self.variance_epsilon)
326
+ return self.weight * hidden_states.to(input_dtype)
327
+
328
+ def extra_repr(self):
329
+ return f"{tuple(self.weight.shape)}, eps={self.variance_epsilon}"
330
+
331
+
332
+class Qwen2_5_VisionBlock(nn.Module):
333
+
334
+ def __init__(
335
+ self,
336
+ dim: int,
337
+ num_heads: int,
338
+ mlp_hidden_dim: int,
339
+ act_fn: Callable[[torch.Tensor], torch.Tensor] = F.silu,
340
+ norm_layer: Optional[Callable[[int], nn.Module]] = None,
341
+ quant_config: Optional[QuantizationConfig] = None,
342
+ prefix: str = "",
343
+ ) -> None:
344
+ super().__init__()
345
+ if norm_layer is None:
346
+ norm_layer = partial(nn.LayerNorm, eps=1e-6)
347
+ self.norm1 = norm_layer(dim)
348
+ self.norm2 = norm_layer(dim)
349
+ self.attn = Qwen2_5_VisionAttention(embed_dim=dim,
350
+ num_heads=num_heads,
351
+ projection_size=dim,
352
+ quant_config=quant_config,
353
+ prefix=f"{prefix}.attn")
354
+ self.mlp = Qwen2_5_VisionMLP(dim,
355
+ mlp_hidden_dim,
356
+ act_fn=act_fn,
357
+ bias=True,
358
+ quant_config=quant_config,
359
+ prefix=f"{prefix}.mlp")
360
+
361
+ def forward(self, x: torch.Tensor, cu_seqlens: torch.Tensor,
362
+ rotary_pos_emb: torch.Tensor) -> torch.Tensor:
363
+ x = x + self.attn(self.norm1(x),
364
+ cu_seqlens=cu_seqlens,
365
+ rotary_pos_emb=rotary_pos_emb)
366
+ x = x + self.mlp(self.norm2(x))
367
+ return x
368
+
369
+
370
+class Qwen2_5_VisionPatchEmbed(nn.Module):
371
+
372
+ def __init__(
373
+ self,
374
+ patch_size: int = 14,
375
+ temporal_patch_size: int = 2,
376
+ in_channels: int = 3,
377
+ hidden_size: int = 1152,
378
+ ) -> None:
379
+ super().__init__()
380
+ self.patch_size = patch_size
381
+ self.temporal_patch_size = temporal_patch_size
382
+ self.hidden_size = hidden_size
383
+
384
+ kernel_size = (temporal_patch_size, patch_size, patch_size)
385
+ self.proj = nn.Conv3d(in_channels,
386
+ hidden_size,
387
+ kernel_size=kernel_size,
388
+ stride=kernel_size,
389
+ bias=False)
390
+
391
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
392
+ L, C = x.shape
393
+ x = x.view(L, -1, self.temporal_patch_size, self.patch_size,
394
+ self.patch_size)
395
+ x = self.proj(x).view(L, self.hidden_size)
396
+ return x
397
+
398
+
399
+class Qwen2_5_VisionPatchMerger(nn.Module):
400
+
401
+ def __init__(
402
+ self,
403
+ d_model: int,
404
+ context_dim: int,
405
+ norm_layer: Optional[Callable[[int], nn.Module]] = None,
406
+ spatial_merge_size: int = 2,
407
+ quant_config: Optional[QuantizationConfig] = None,
408
+ prefix: str = "",
409
+ ) -> None:
410
+ super().__init__()
411
+ self.hidden_size = context_dim * (spatial_merge_size**2)
412
+ if norm_layer is None:
413
+ norm_layer = partial(nn.LayerNorm, eps=1e-6)
414
+ self.ln_q = norm_layer(context_dim)
415
+ self.mlp = nn.ModuleList([
416
+ ColumnParallelLinear(self.hidden_size,
417
+ self.hidden_size,
418
+ bias=True,
419
+ quant_config=quant_config,
420
+ prefix=f"{prefix}.mlp.0"),
421
+ nn.GELU(),
422
+ RowParallelLinear(self.hidden_size,
423
+ d_model,
424
+ bias=True,
425
+ quant_config=quant_config,
426
+ prefix=f"{prefix}.mlp.2"),
427
+ ])
428
+
429
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
430
+ x = self.ln_q(x)
431
+ x = x.view(-1, self.hidden_size)
432
+
433
+ mlp_fc1, mlp_act, mlp_fc2 = self.mlp
434
+ x_parallel, _ = mlp_fc1(x)
435
+ x_parallel = mlp_act(x_parallel)
436
+ out, _ = mlp_fc2(x_parallel)
437
+ return out
438
+
439
+
440
+class Qwen2_5_VisionRotaryEmbedding(nn.Module):
441
+
442
+ def __init__(self, dim: int, theta: float = 10000.0) -> None:
443
+ super().__init__()
444
+ self.dim = dim
445
+ self.theta = theta
446
+ inv_freq = 1.0 / (theta
447
+ **(torch.arange(0, dim, 2, dtype=torch.float) / dim))
448
+ self.register_buffer("inv_freq", inv_freq, persistent=False)
449
+ self._seq_len_cached = 0
450
+ self._freqs_cached = None
451
+
452
+ def update_freqs_cache(self, seqlen: int) -> None:
453
+ if seqlen > self._seq_len_cached:
454
+ seqlen *= 2
455
+ self._seq_len_cached = seqlen
456
+ self.inv_freq = 1.0 / (self.theta**(torch.arange(
457
+ 0, self.dim, 2, dtype=torch.float, device=self.inv_freq.device)
458
+ / self.dim))
459
+ seq = torch.arange(seqlen,
460
+ device=self.inv_freq.device,
461
+ dtype=self.inv_freq.dtype)
462
+ freqs = torch.outer(seq, self.inv_freq)
463
+ self._freqs_cached = freqs
464
+
465
+ def forward(self, seqlen: int) -> torch.Tensor:
466
+ self.update_freqs_cache(seqlen)
467
+ return self._freqs_cached[:seqlen]
468
+
469
+
470
+class Qwen2_5_VisionTransformer(nn.Module):
471
+
472
+ def __init__(
473
+ self,
474
+ vision_config: Qwen2_5_VLVisionConfig,
475
+ norm_eps: float = 1e-6,
476
+ quant_config: Optional[QuantizationConfig] = None,
477
+ prefix: str = "",
478
+ ) -> None:
479
+ super().__init__()
480
+
481
+ patch_size = vision_config.patch_size
482
+ temporal_patch_size = vision_config.temporal_patch_size
483
+ in_channels = vision_config.in_channels
484
+ depth = vision_config.depth
485
+ self.hidden_size = vision_config.hidden_size
486
+ self.num_heads = vision_config.num_heads
487
+
488
+ # args for get_window_index
489
+ self.window_size = vision_config.window_size
490
+ self.patch_size = vision_config.patch_size
491
+ self.spatial_merge_size = vision_config.spatial_merge_size
492
+ self.fullatt_block_indexes = vision_config.fullatt_block_indexes
493
+ self.spatial_merge_unit = self.spatial_merge_size**2
494
+
495
+ self.patch_embed = Qwen2_5_VisionPatchEmbed(
496
+ patch_size=patch_size,
497
+ temporal_patch_size=temporal_patch_size,
498
+ in_channels=in_channels,
499
+ hidden_size=self.hidden_size,
500
+ )
501
+
502
+ # NOTE: We use torch native RMSNorm here for precision purposes.
503
+ norm_layer = partial(Qwen2RMSNorm, eps=norm_eps)
504
+ head_dim = self.hidden_size // self.num_heads
505
+ self.rotary_pos_emb = Qwen2_5_VisionRotaryEmbedding(head_dim // 2)
506
+
507
+ self.blocks = nn.ModuleList([
508
+ Qwen2_5_VisionBlock(
509
+ dim=self.hidden_size,
510
+ num_heads=self.num_heads,
511
+ mlp_hidden_dim=vision_config.intermediate_size,
512
+ act_fn=_ACTIVATION_REGISTRY[vision_config.hidden_act],
513
+ norm_layer=norm_layer,
514
+ quant_config=quant_config,
515
+ prefix=f"{prefix}.blocks.{layer_idx}")
516
+ for layer_idx in range(depth)
517
+ ])
518
+ self.merger = Qwen2_5_VisionPatchMerger(
519
+ d_model=vision_config.out_hidden_size,
520
+ context_dim=self.hidden_size,
521
+ norm_layer=norm_layer,
522
+ spatial_merge_size=self.spatial_merge_size,
523
+ quant_config=quant_config,
524
+ prefix=f"{prefix}.merger",
525
+ )
526
+
527
+ @property
528
+ def dtype(self) -> torch.dtype:
529
+ return self.patch_embed.proj.weight.dtype
530
+
531
+ @property
532
+ def device(self) -> torch.device:
533
+ return self.patch_embed.proj.weight.device
534
+
535
+ def rot_pos_emb(self, grid_thw: torch.Tensor) -> torch.Tensor:
536
+ pos_ids = []
537
+ for t, h, w in grid_thw:
538
+ hpos_ids = torch.arange(h).unsqueeze(1).expand(-1, w)
539
+ wpos_ids = torch.arange(w).unsqueeze(0).expand(h, -1)
540
+ hpos_ids = hpos_ids.reshape(
541
+ h // self.spatial_merge_size,
542
+ self.spatial_merge_size,
543
+ w // self.spatial_merge_size,
544
+ self.spatial_merge_size,
545
+ ).permute(0, 2, 1, 3).flatten()
546
+ wpos_ids = wpos_ids.reshape(
547
+ h // self.spatial_merge_size,
548
+ self.spatial_merge_size,
549
+ w // self.spatial_merge_size,
550
+ self.spatial_merge_size,
551
+ ).permute(0, 2, 1, 3).flatten()
552
+ pos_ids.append(
553
+ torch.stack([hpos_ids, wpos_ids], dim=-1).repeat(t, 1))
554
+ pos_ids = torch.cat(pos_ids, dim=0)
555
+ max_grid_size = grid_thw[:, 1:].max()
556
+ rotary_pos_emb_full = self.rotary_pos_emb(max_grid_size)
557
+ rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1)
558
+ return rotary_pos_emb
559
+
560
+ def get_window_index(self, grid_thw):
561
+ window_index: list = []
562
+ cu_window_seqlens: list = [0]
563
+ window_index_id = 0
564
+ vit_merger_window_size = (self.window_size //
565
+ self.spatial_merge_size // self.patch_size)
566
+
567
+ for grid_t, grid_h, grid_w in grid_thw:
568
+ llm_grid_h = grid_h // self.spatial_merge_size
569
+ llm_grid_w = grid_w // self.spatial_merge_size
570
+ index = torch.arange(grid_t * llm_grid_h * llm_grid_w).reshape(
571
+ grid_t, llm_grid_h, llm_grid_w)
572
+ pad_h = vit_merger_window_size - llm_grid_h % vit_merger_window_size
573
+ pad_w = vit_merger_window_size - llm_grid_w % vit_merger_window_size
574
+ num_windows_h = (llm_grid_h + pad_h) // vit_merger_window_size
575
+ num_windows_w = (llm_grid_w + pad_w) // vit_merger_window_size
576
+ index_padded = F.pad(index, (0, pad_w, 0, pad_h), 'constant', -100)
577
+ index_padded = index_padded.reshape(grid_t, num_windows_h,
578
+ vit_merger_window_size,
579
+ num_windows_w,
580
+ vit_merger_window_size)
581
+ index_padded = index_padded.permute(0, 1, 3, 2, 4).reshape(
582
+ grid_t, num_windows_h * num_windows_w, vit_merger_window_size,
583
+ vit_merger_window_size)
584
+ seqlens = (index_padded != -100).sum([2, 3]).reshape(-1)
585
+ index_padded = index_padded.reshape(-1)
586
+ index_new = index_padded[index_padded != -100]
587
+ window_index.append(index_new + window_index_id)
588
+ cu_seqlens_tmp = seqlens.cumsum(
589
+ 0) * self.spatial_merge_unit + cu_window_seqlens[-1]
590
+ cu_window_seqlens.extend(cu_seqlens_tmp.tolist())
591
+ window_index_id += (grid_t * llm_grid_h * llm_grid_w).item()
592
+ window_index = torch.cat(window_index, dim=0)
593
+ return window_index, cu_window_seqlens
594
+
595
+ def forward(
596
+ self,
597
+ x: torch.Tensor,
598
+ grid_thw: torch.Tensor,
599
+ ) -> torch.Tensor:
600
+ # patchify
601
+ hidden_states = x.to(device=self.device, dtype=self.dtype)
602
+ hidden_states = self.patch_embed(hidden_states)
603
+
604
+ # compute position embedding
605
+ rotary_pos_emb = self.rot_pos_emb(grid_thw)
606
+
607
+ # windows attention
608
+ window_index, cu_window_seqlens = self.get_window_index(grid_thw)
609
+ cu_window_seqlens = torch.tensor(
610
+ cu_window_seqlens,
611
+ device=hidden_states.device,
612
+ dtype=grid_thw.dtype if torch.jit.is_tracing() else torch.int32)
613
+ cu_window_seqlens = torch.unique_consecutive(cu_window_seqlens)
614
+ seq_len, _ = hidden_states.size()
615
+ hidden_states = hidden_states.reshape(
616
+ seq_len // self.spatial_merge_unit, self.spatial_merge_unit, -1)
617
+ hidden_states = hidden_states[window_index, :, :]
618
+ hidden_states = hidden_states.reshape(seq_len, -1)
619
+ rotary_pos_emb = rotary_pos_emb.reshape(
620
+ seq_len // self.spatial_merge_unit, self.spatial_merge_unit, -1)
621
+ rotary_pos_emb = rotary_pos_emb[window_index, :, :]
622
+ rotary_pos_emb = rotary_pos_emb.reshape(seq_len, -1)
623
+ # compute cu_seqlens
624
+ cu_seqlens = torch.repeat_interleave(grid_thw[:, 1] * grid_thw[:, 2],
625
+ grid_thw[:, 0]).cumsum(
626
+ dim=0, dtype=torch.int32)
627
+ cu_seqlens = F.pad(cu_seqlens, (1, 0), "constant", 0)
628
+
629
+ # transformers
630
+ hidden_states = hidden_states.unsqueeze(1)
631
+ for layer_num, blk in enumerate(self.blocks):
632
+ if layer_num in self.fullatt_block_indexes:
633
+ cu_seqlens_now = cu_seqlens
634
+ else:
635
+ cu_seqlens_now = cu_window_seqlens
636
+ hidden_states = blk(hidden_states,
637
+ cu_seqlens=cu_seqlens_now,
638
+ rotary_pos_emb=rotary_pos_emb)
639
+
640
+ # adapter
641
+ hidden_states = self.merger(hidden_states)
642
+ reverse_indices = torch.argsort(window_index)
643
+ hidden_states = hidden_states[reverse_indices, :]
644
+ return hidden_states
645
+
646
+ def load_weights(self, weights: Iterable[Tuple[str,
647
+ torch.Tensor]]) -> Set[str]:
648
+ stacked_params_mapping = [
649
+ # (param_name, shard_name, shard_id)
650
+ ("qkv_proj", "q_proj", "q"),
651
+ ("qkv_proj", "k_proj", "k"),
652
+ ("qkv_proj", "v_proj", "v"),
653
+ ]
654
+ params_dict = dict(self.named_parameters(remove_duplicate=False))
655
+ loaded_params: Set[str] = set()
656
+
657
+ for name, loaded_weight in weights:
658
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
659
+ if weight_name not in name:
660
+ continue
661
+ name = name.replace(weight_name, param_name)
662
+
663
+ param = params_dict[name]
664
+ weight_loader = param.weight_loader
665
+ weight_loader(param, loaded_weight, shard_id)
666
+ break
667
+ else:
668
+ if name.endswith("qkv.weight"):
669
+ visual_num_heads = self.num_heads
670
+ visual_embed_dim = self.hidden_size
671
+ head_size = visual_embed_dim // visual_num_heads
672
+ loaded_weight = loaded_weight.view(3, visual_num_heads,
673
+ head_size,
674
+ visual_embed_dim)
675
+ loaded_weight = loaded_weight.transpose(0, 1)
676
+ loaded_weight = loaded_weight.reshape(-1, visual_embed_dim)
677
+ elif name.endswith("qkv.bias"):
678
+ visual_num_heads = self.num_heads
679
+ visual_embed_dim = self.hidden_size
680
+ head_size = visual_embed_dim // visual_num_heads
681
+ loaded_weight = loaded_weight.view(3, visual_num_heads,
682
+ head_size)
683
+ loaded_weight = loaded_weight.transpose(0, 1)
684
+ loaded_weight = loaded_weight.reshape(-1)
685
+
686
+ param = params_dict[name]
687
+ weight_loader = getattr(param, "weight_loader",
688
+ default_weight_loader)
689
+ weight_loader(param, loaded_weight)
690
+ loaded_params.add(name)
691
+ return loaded_params
692
+
693
+
694
+class Qwen2_5_VLProcessingInfo(Qwen2VLProcessingInfo):
695
+
696
+ def get_hf_config(self):
697
+ return self.ctx.get_hf_config(Qwen2_5_VLConfig)
698
+
699
+ def get_hf_processor(
700
+ self,
701
+ *,
702
+ min_pixels: Optional[int] = None,
703
+ max_pixels: Optional[int] = None,
704
+ fps: Optional[float] = 2.0,
705
+ ) -> Qwen2_5_VLProcessor:
706
+ hf_processor = self.ctx.get_hf_processor(Qwen2_5_VLProcessor)
707
+ image_processor = hf_processor.image_processor # type: ignore
708
+ assert isinstance(image_processor, Qwen2_5_VLImageProcessor)
709
+
710
+ if min_pixels:
711
+ image_processor.min_pixels = min_pixels
712
+ if max_pixels:
713
+ image_processor.max_pixels = max_pixels
714
+ if max_pixels or min_pixels:
715
+ image_processor.size = {
716
+ "min_pixels": image_processor.min_pixels,
717
+ "max_pixels": image_processor.max_pixels,
718
+ }
719
+
720
+ return hf_processor
721
+
722
+ def get_image_processor(
723
+ self,
724
+ *,
725
+ min_pixels: Optional[int] = None,
726
+ max_pixels: Optional[int] = None,
727
+ fps: Optional[float] = 2.0,
728
+ ) -> Qwen2_5_VLImageProcessor:
729
+ hf_processor = self.get_hf_processor(
730
+ min_pixels=min_pixels,
731
+ max_pixels=max_pixels,
732
+ fps=fps,
733
+ )
734
+ image_processor = hf_processor.image_processor # type: ignore
735
+ assert isinstance(image_processor, Qwen2_5_VLImageProcessor)
736
+ return image_processor
737
+
738
+
739
+class Qwen2_5_VLMultiModalProcessor(Qwen2VLMultiModalProcessor):
740
+
741
+ def _get_mm_fields_config(
742
+ self,
743
+ hf_inputs: BatchFeature,
744
+ hf_processor_mm_kwargs: Mapping[str, object],
745
+ ) -> Mapping[str, MultiModalFieldConfig]:
746
+ return dict(
747
+ **super()._get_mm_fields_config(hf_inputs, hf_processor_mm_kwargs),
748
+ second_per_grid_ts=MultiModalFieldConfig.batched("video"),
749
+ )
750
+
751
+
752
+@MULTIMODAL_REGISTRY.register_processor(
753
+ Qwen2_5_VLMultiModalProcessor,
754
+ info=Qwen2_5_VLProcessingInfo,
755
+ dummy_inputs=Qwen2_5_VLDummyInputsBuilder)
756
+class Qwen2_5_VLForConditionalGeneration(nn.Module, SupportsMultiModal,
757
+ SupportsLoRA, SupportsPP):
758
+ packed_modules_mapping = {
759
+ "qkv_proj": [
760
+ "q_proj",
761
+ "k_proj",
762
+ "v_proj",
763
+ ]
764
+ }
765
+
766
+ # LoRA specific attributes, TODO: double check
767
+ supported_lora_modules = [
768
+ "qkv_proj",
769
+ "o_proj",
770
+ "gate_up_proj",
771
+ "down_proj",
772
+ "gate_proj"
773
+ "up_proj",
774
+ # vision tower
775
+ "qkv",
776
+ "attn.proj", # Distinguish patch_embed.proj
777
+ "fc1",
778
+ "fc2",
779
+ # projector
780
+ "mlp.0",
781
+ "mlp.2"
782
+ ]
783
+ embedding_modules = {}
784
+ embedding_padding_modules = []
785
+
786
+ # To ensure correct weight loading and mapping.
787
+ hf_to_vllm_mapper = WeightsMapper(orig_to_new_prefix={
788
+ "lm_head.": "language_model.lm_head.",
789
+ "model.": "language_model.model.",
790
+ })
791
+
792
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
793
+ super().__init__()
794
+ config: Qwen2_5_VLConfig = vllm_config.model_config.hf_config
795
+ quant_config = vllm_config.quant_config
796
+ multimodal_config = vllm_config.model_config.multimodal_config
797
+
798
+ self.config = config
799
+ self.multimodal_config = multimodal_config
800
+
801
+ self.visual = Qwen2_5_VisionTransformer(
802
+ config.vision_config,
803
+ norm_eps=getattr(config, "rms_norm_eps", 1e-6),
804
+ quant_config=self._maybe_ignore_quant_config(quant_config),
805
+ prefix=maybe_prefix(prefix, "visual"),
806
+ )
807
+
808
+ self.language_model = init_vllm_registered_model(
809
+ vllm_config=vllm_config,
810
+ prefix=maybe_prefix(prefix, "language_model"),
811
+ architectures=["Qwen2ForCausalLM"],
812
+ )
813
+
814
+ self.make_empty_intermediate_tensors = (
815
+ self.language_model.make_empty_intermediate_tensors)
816
+
817
+ @cached_property
818
+ def sampler(self):
819
+ if hasattr(self.language_model, "sampler"):
820
+ return self.language_model.sampler
821
+
822
+ return get_sampler()
823
+
824
+ def _maybe_ignore_quant_config(self, quant_config: QuantizationConfig):
825
+ # GPTQ configs do not have a list of ignored modules, however AutoGPTQ
826
+ # seems to avoid vision encoder sections for some models.
827
+ if isinstance(quant_config, (GPTQConfig, GPTQMarlinConfig)):
828
+ return None
829
+ return quant_config
830
+
831
+ def _validate_and_reshape_mm_tensor(self, mm_input: object,
832
+ name: str) -> torch.Tensor:
833
+ if not isinstance(mm_input, (torch.Tensor, list)):
834
+ raise ValueError(f"Incorrect type of {name}. "
835
+ f"Got type: {type(mm_input)}")
836
+ if isinstance(mm_input, torch.Tensor):
837
+ if mm_input.ndim == 2:
838
+ return mm_input
839
+ if mm_input.ndim != 3:
840
+ raise ValueError(f"{name} should be 2D or batched 3D tensor. "
841
+ f"Got ndim: {mm_input.ndim} "
842
+ f"(shape={mm_input.shape})")
843
+ return torch.concat(list(mm_input))
844
+ else:
845
+ return torch.concat(mm_input)
846
+
847
+ def _parse_and_validate_image_input(
848
+ self, **kwargs: object) -> Optional[Qwen2_5_VLImageInputs]:
849
+ pixel_values = kwargs.pop("pixel_values", None)
850
+ image_embeds = kwargs.pop("image_embeds", None)
851
+ image_grid_thw = kwargs.pop("image_grid_thw", None)
852
+
853
+ if pixel_values is None and image_embeds is None:
854
+ return None
855
+
856
+ if pixel_values is not None:
857
+ pixel_values = self._validate_and_reshape_mm_tensor(
858
+ pixel_values, "image pixel values")
859
+ image_grid_thw = self._validate_and_reshape_mm_tensor(
860
+ image_grid_thw, "image grid_thw")
861
+
862
+ if not isinstance(pixel_values, (torch.Tensor, list)):
863
+ raise ValueError("Incorrect type of image pixel values. "
864
+ f"Got type: {type(pixel_values)}")
865
+
866
+ return Qwen2_5_VLImagePixelInputs(type="pixel_values",
867
+ pixel_values=pixel_values,
868
+ image_grid_thw=image_grid_thw)
869
+
870
+ if image_embeds is not None:
871
+ image_embeds = self._validate_and_reshape_mm_tensor(
872
+ image_embeds, "image embeds")
873
+ image_grid_thw = self._validate_and_reshape_mm_tensor(
874
+ image_grid_thw, "image grid_thw")
875
+
876
+ if not isinstance(image_embeds, torch.Tensor):
877
+ raise ValueError("Incorrect type of image embeddings. "
878
+ f"Got type: {type(image_embeds)}")
879
+ return Qwen2_5_VLImageEmbeddingInputs(
880
+ type="image_embeds",
881
+ image_embeds=image_embeds,
882
+ image_grid_thw=image_grid_thw)
883
+
884
+ def _parse_and_validate_video_input(
885
+ self, **kwargs: object) -> Optional[Qwen2_5_VLVideoInputs]:
886
+ pixel_values_videos = kwargs.pop("pixel_values_videos", None)
887
+ video_embeds = kwargs.pop("video_embeds", None)
888
+ video_grid_thw = kwargs.pop("video_grid_thw", None)
889
+ second_per_grid_ts = kwargs.pop("second_per_grid_ts", None)
890
+
891
+ if pixel_values_videos is None and video_embeds is None:
892
+ return None
893
+
894
+ if pixel_values_videos is not None:
895
+ pixel_values_videos = self._validate_and_reshape_mm_tensor(
896
+ pixel_values_videos, "video pixel values")
897
+ video_grid_thw = self._validate_and_reshape_mm_tensor(
898
+ video_grid_thw, "video grid_thw")
899
+
900
+ return Qwen2_5_VLVideoPixelInputs(
901
+ type="pixel_values_videos",
902
+ pixel_values_videos=pixel_values_videos,
903
+ video_grid_thw=video_grid_thw,
904
+ second_per_grid_ts=second_per_grid_ts,
905
+ )
906
+
907
+ if video_embeds is not None:
908
+ video_embeds = self._validate_and_reshape_mm_tensor(
909
+ video_embeds, "video embeds")
910
+ video_grid_thw = self._validate_and_reshape_mm_tensor(
911
+ video_grid_thw, "video grid_thw")
912
+
913
+ if not isinstance(video_embeds, torch.Tensor):
914
+ raise ValueError("Incorrect type of video embeddings. "
915
+ f"Got type: {type(video_embeds)}")
916
+ return Qwen2_5_VLVideoEmbeddingInputs(
917
+ type="video_embeds",
918
+ video_embeds=video_embeds,
919
+ video_grid_thw=video_grid_thw)
920
+
921
+ def _process_image_input(
922
+ self,
923
+ image_input: Qwen2_5_VLImageInputs) -> tuple[torch.Tensor, ...]:
924
+
925
+ grid_thw = image_input["image_grid_thw"]
926
+ assert grid_thw.ndim == 2
927
+
928
+ if image_input["type"] == "image_embeds":
929
+ image_embeds = image_input["image_embeds"].type(self.visual.dtype)
930
+ else:
931
+ pixel_values = image_input["pixel_values"].type(self.visual.dtype)
932
+ image_embeds = self.visual(pixel_values, grid_thw=grid_thw)
933
+
934
+ # Split concatenated embeddings for each image item.
935
+ merge_size = self.visual.spatial_merge_size
936
+ sizes = grid_thw.prod(-1) // merge_size // merge_size
937
+
938
+ return image_embeds.split(sizes.tolist())
939
+
940
+ def _process_video_input(
941
+ self,
942
+ video_input: Qwen2_5_VLVideoInputs) -> tuple[torch.Tensor, ...]:
943
+
944
+ grid_thw = video_input["video_grid_thw"]
945
+ assert grid_thw.ndim == 2
946
+
947
+ if video_input["type"] == "video_embeds":
948
+ video_embeds = video_input["video_embeds"].type(self.visual.dtype)
949
+ else:
950
+ pixel_values_videos = video_input["pixel_values_videos"].type(
951
+ self.visual.dtype)
952
+ video_embeds = self.visual(pixel_values_videos, grid_thw=grid_thw)
953
+
954
+ # Split concatenated embeddings for each video item.
955
+ merge_size = self.visual.spatial_merge_size
956
+ sizes = grid_thw.prod(-1) // merge_size // merge_size
957
+
958
+ return video_embeds.split(sizes.tolist())
959
+
960
+ def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
961
+ modalities = {}
962
+
963
+ # Preserve the order of modalities if there are multiple of them
964
+ # from the order of kwargs.
965
+ for input_key in kwargs:
966
+ if input_key in ("pixel_values",
967
+ "image_embeds") and "images" not in modalities:
968
+ modalities["images"] = self._parse_and_validate_image_input(
969
+ **kwargs)
970
+ if input_key in ("pixel_values_videos",
971
+ "video_embeds") and "videos" not in modalities:
972
+ modalities["videos"] = self._parse_and_validate_video_input(
973
+ **kwargs)
974
+ return modalities
975
+
976
+ def get_multimodal_embeddings(
977
+ self, **kwargs) -> Optional[tuple[torch.Tensor, ...]]:
978
+
979
+ modalities = self._parse_and_validate_multimodal_inputs(**kwargs)
980
+ if not modalities:
981
+ return None
982
+
983
+ # The result multimodal_embeddings is tuple of tensors, with each
984
+ # tensor correspoending to a multimodal data item (image or video).
985
+ multimodal_embeddings: tuple[torch.Tensor, ...] = ()
986
+
987
+ # NOTE: It is important to iterate over the keys in this dictionary
988
+ # to preserve the order of the modalities.
989
+ for modality in modalities:
990
+ if modality == "images":
991
+ image_input = modalities["images"]
992
+ vision_embeddings = self._process_image_input(image_input)
993
+ multimodal_embeddings += vision_embeddings
994
+ if modality == "videos":
995
+ video_input = modalities["videos"]
996
+ video_embeddings = self._process_video_input(video_input)
997
+ multimodal_embeddings += video_embeddings
998
+ return multimodal_embeddings
999
+
1000
+ def get_input_embeddings(
1001
+ self,
1002
+ input_ids: torch.Tensor,
1003
+ multimodal_embeddings: Optional[tuple[torch.Tensor, ...]] = None,
1004
+ ) -> torch.Tensor:
1005
+ inputs_embeds = self.language_model.get_input_embeddings(input_ids)
1006
+ if multimodal_embeddings is not None:
1007
+ inputs_embeds = merge_multimodal_embeddings(
1008
+ input_ids, inputs_embeds, multimodal_embeddings,
1009
+ [self.config.image_token_id, self.config.video_token_id])
1010
+ return inputs_embeds
1011
+
1012
+ def get_input_embeddings_v0(
1013
+ self,
1014
+ input_ids: torch.Tensor,
1015
+ image_input: Optional[tuple[torch.Tensor, ...]] = None,
1016
+ video_input: Optional[tuple[torch.Tensor, ...]] = None,
1017
+ ) -> torch.Tensor:
1018
+
1019
+ inputs_embeds = self.get_input_embeddings(input_ids)
1020
+ if image_input is not None:
1021
+ image_embeds = self._process_image_input(image_input)
1022
+ inputs_embeds = merge_multimodal_embeddings(
1023
+ input_ids,
1024
+ inputs_embeds,
1025
+ image_embeds,
1026
+ placeholder_token_id=self.config.image_token_id,
1027
+ )
1028
+
1029
+ if video_input is not None:
1030
+ video_embeds = self._process_video_input(video_input)
1031
+ inputs_embeds = merge_multimodal_embeddings(
1032
+ input_ids,
1033
+ inputs_embeds,
1034
+ video_embeds,
1035
+ placeholder_token_id=self.config.video_token_id,
1036
+ )
1037
+ return inputs_embeds
1038
+
1039
+ def forward(
1040
+ self,
1041
+ input_ids: torch.Tensor,
1042
+ positions: torch.Tensor,
1043
+ kv_caches: List[torch.Tensor],
1044
+ attn_metadata: AttentionMetadata,
1045
+ intermediate_tensors: Optional[IntermediateTensors] = None,
1046
+ inputs_embeds: Optional[torch.Tensor] = None,
1047
+ **kwargs: object,
1048
+ ) -> Union[torch.Tensor, IntermediateTensors]:
1049
+ """Run forward pass for Qwen2.5-VL.
1050
+
1051
+ Args:
1052
+ input_ids: Flattened (concatenated) input_ids corresponding to a
1053
+ batch.
1054
+ positions: Flattened (concatenated) position ids corresponding to a
1055
+ batch.
1056
+ **NOTE**: If mrope is enabled (default setting for Qwen2.5-VL
1057
+ opensource models), the shape will be `(3, seq_len)`,
1058
+ otherwise it will be `(seq_len,).
1059
+ pixel_values: Pixel values to be fed to a model.
1060
+ `None` if no images are passed.
1061
+ image_grid_thw: Tensor `(n_images, 3)` of image 3D grid in LLM.
1062
+ `None` if no images are passed.
1063
+ pixel_values_videos: Pixel values of videos to be fed to a model.
1064
+ `None` if no videos are passed.
1065
+ video_grid_thw: Tensor `(n_videos, 3)` of video 3D grid in LLM.
1066
+ `None` if no videos are passed.
1067
+ second_per_grid_ts: Tensor `(num_videos)` of video time interval (
1068
+ in seconds) for each grid along the temporal dimension in the
1069
+ 3D position IDs. `None` if no videos are passed.
1070
+ """
1071
+
1072
+ if intermediate_tensors is not None:
1073
+ inputs_embeds = None
1074
+
1075
+ # NOTE: In v1, inputs_embeds is always generated at model runner from
1076
+ # `get_multimodal_embeddings` and `get_input_embeddings`, this
1077
+ # condition is only for v0 compatibility.
1078
+ elif inputs_embeds is None:
1079
+ image_input = self._parse_and_validate_image_input(**kwargs)
1080
+ video_input = self._parse_and_validate_video_input(**kwargs)
1081
+
1082
+ if image_input is None and video_input is None:
1083
+ inputs_embeds = None
1084
+ else:
1085
+ if uses_mrope(self.config):
1086
+ assert positions.ndim == 2 and positions.size(0) == 3, (
1087
+ "multimodal section rotary embedding requires "
1088
+ f"(3, seq_len) positions, but got {positions.size()}")
1089
+ inputs_embeds = self.get_input_embeddings_v0(
1090
+ input_ids,
1091
+ image_input=image_input,
1092
+ video_input=video_input)
1093
+ input_ids = None
1094
+
1095
+ hidden_states = self.language_model.model(
1096
+ input_ids=input_ids,
1097
+ positions=positions,
1098
+ kv_caches=kv_caches,
1099
+ attn_metadata=attn_metadata,
1100
+ intermediate_tensors=intermediate_tensors,
1101
+ inputs_embeds=inputs_embeds,
1102
+ )
1103
+ return hidden_states
1104
+
1105
+ def compute_logits(
1106
+ self,
1107
+ hidden_states: torch.Tensor,
1108
+ sampling_metadata: SamplingMetadata,
1109
+ ) -> Optional[torch.Tensor]:
1110
+ return self.language_model.compute_logits(hidden_states,
1111
+ sampling_metadata)
1112
+
1113
+ def sample(
1114
+ self,
1115
+ logits: torch.Tensor,
1116
+ sampling_metadata: SamplingMetadata,
1117
+ ) -> Optional[SamplerOutput]:
1118
+ return self.language_model.sample(logits, sampling_metadata)
1119
+
1120
+ def load_weights(self, weights: Iterable[Tuple[str,
1121
+ torch.Tensor]]) -> Set[str]:
1122
+
1123
+ loader = AutoWeightsLoader(self)
1124
+ return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
1125
+
1126
+ def get_mm_mapping(self) -> MultiModelKeys:
1127
+ """
1128
+ Get the module prefix in multimodal models
1129
+ """
1130
+ return MultiModelKeys.from_string_field(
1131
+ language_model="language_model",
1132
+ connector="visual.",
1133
+ tower_model="visual.merger.")
vllm/model_executor/models/qwen2_vl.py
+8
-8
@@ -650,8 +650,8 @@ class Qwen2VisionTransformer(nn.Module):
650
return loaded_params
651
652
653
-class Qwen2EmbeddingItems(ModalityDataItems[dict[str, torch.Tensor],
654
- dict[str, torch.Tensor]]):
653
+class Qwen2VLEmbeddingItems(ModalityDataItems[dict[str, torch.Tensor],
654
+ dict[str, torch.Tensor]]):
655
656
def __init__(self, data: dict, modality: str) -> None:
657
super().__init__(data, modality)
@@ -683,26 +683,26 @@ class Qwen2EmbeddingItems(ModalityDataItems[dict[str, torch.Tensor],
683
return self.data
684
685
686
-class Qwen2ImageEmbeddingItems(Qwen2EmbeddingItems):
686
+class Qwen2VLImageEmbeddingItems(Qwen2VLEmbeddingItems):
687
688
def __init__(self, data: dict) -> None:
689
super().__init__(data, "image")
690
691
692
-class Qwen2VideoEmbeddingItems(Qwen2EmbeddingItems):
692
+class Qwen2VLVideoEmbeddingItems(Qwen2VLEmbeddingItems):
693
694
def __init__(self, data: dict) -> None:
695
super().__init__(data, "video")
696
697
698
-class Qwen2MultiModalDataParser(MultiModalDataParser):
698
+class Qwen2VLMultiModalDataParser(MultiModalDataParser):
699
700
def _parse_image_data(
701
self,
702
data: Union[dict[str, torch.Tensor], ModalityData[ImageItem]],
703
) -> ModalityDataItems[Any, Any]:
704
if isinstance(data, dict):
705
- return Qwen2EmbeddingItems(data, modality="image")
705
+ return Qwen2VLEmbeddingItems(data, modality="image")
706
707
return super()._parse_image_data(data)
708
@@ -711,7 +711,7 @@ class Qwen2MultiModalDataParser(MultiModalDataParser):
711
data: Union[dict[str, torch.Tensor], ModalityData[VideoItem]],
712
) -> ModalityDataItems[Any, Any]:
713
if isinstance(data, dict):
714
- return Qwen2EmbeddingItems(data, modality="video")
714
+ return Qwen2VLEmbeddingItems(data, modality="video")
715
716
return super()._parse_video_data(data)
717
@@ -948,7 +948,7 @@ class Qwen2VLMultiModalProcessor(BaseMultiModalProcessor[Qwen2VLProcessingInfo]
948
):
949
950
def _get_data_parser(self) -> MultiModalDataParser:
951
- return Qwen2MultiModalDataParser()
951
+ return Qwen2VLMultiModalDataParser()
952
953
def _get_prompt_replacements(
954
self,
vllm/model_executor/models/registry.py
+1
@@ -172,6 +172,7 @@ _MULTIMODAL_MODELS = {
172
"PixtralForConditionalGeneration": ("pixtral", "PixtralForConditionalGeneration"), # noqa: E501
173
"QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
174
"Qwen2VLForConditionalGeneration": ("qwen2_vl", "Qwen2VLForConditionalGeneration"), # noqa: E501
175
+ "Qwen2_5_VLForConditionalGeneration": ("qwen2_5_vl", "Qwen2_5_VLForConditionalGeneration"), # noqa: E501
176
"Qwen2AudioForConditionalGeneration": ("qwen2_audio", "Qwen2AudioForConditionalGeneration"), # noqa: E501
177
"UltravoxModel": ("ultravox", "UltravoxModel"),
178
# [Encoder-decoder]
vllm/v1/worker/gpu_model_runner.py
+6
-6
@@ -285,6 +285,7 @@ class GPUModelRunner:
285
if self.model_config.uses_mrope:
286
image_grid_thw = []
287
video_grid_thw = []
288
+ second_per_grid_ts = []
289
for mm_input in self.requests[req_id].mm_inputs:
290
if mm_input.get("image_grid_thw") is not None:
291
image_grid_thw.extend(
@@ -292,6 +293,9 @@ class GPUModelRunner:
293
if mm_input.get("video_grid_thw") is not None:
294
video_grid_thw.extend(
295
mm_input["video_grid_thw"].tolist())
296
+ if mm_input.get("second_per_grid_ts") is not None:
297
+ second_per_grid_ts.extend(
298
+ mm_input["second_per_grid_ts"])
299
300
hf_config = self.model_config.hf_config
301
@@ -299,14 +303,10 @@ class GPUModelRunner:
303
self.requests[req_id].mrope_position_delta = \
304
MRotaryEmbedding.get_input_positions_tensor(
305
self.requests[req_id].prompt_token_ids,
306
+ hf_config=hf_config,
307
image_grid_thw=image_grid_thw,
308
video_grid_thw=video_grid_thw,
304
- image_token_id=hf_config.image_token_id,
305
- video_token_id=hf_config.video_token_id,
306
- vision_start_token_id=hf_config.vision_start_token_id,
307
- vision_end_token_id=hf_config.vision_end_token_id,
308
- spatial_merge_size=hf_config.vision_config.
309
- spatial_merge_size,
309
+ second_per_grid_ts=second_per_grid_ts,
310
)
311
312
req_ids_to_add.append(req_id)
vllm/worker/cpu_model_runner.py
+3
-6
@@ -386,20 +386,17 @@ class ModelInputForCPUBuilder(ModelRunnerInputBuilderBase[ModelInputForCPU]):
386
"mrope embedding type requires multi-modal input mapper "
387
"returns 'image_grid_thw' or 'video_grid_thw'.")
388
389
+ second_per_grid_ts = mm_kwargs.get("second_per_grid_ts", None)
390
hf_config = self.runner.model_config.hf_config
391
token_ids = seq_data.get_token_ids()
392
393
mrope_positions, mrope_position_delta = \
394
MRotaryEmbedding.get_input_positions(
395
token_ids,
396
+ hf_config=hf_config,
397
image_grid_thw=image_grid_thw,
398
video_grid_thw=video_grid_thw,
397
- image_token_id=hf_config.image_token_id,
398
- video_token_id=hf_config.video_token_id,
399
- vision_start_token_id=hf_config.vision_start_token_id,
400
- vision_end_token_id=hf_config.vision_end_token_id,
401
- spatial_merge_size=hf_config.vision_config.
402
- spatial_merge_size,
399
+ second_per_grid_ts=second_per_grid_ts,
400
context_len=computed_len,
401
)
402
seq_data.mrope_position_delta = mrope_position_delta
vllm/worker/model_runner.py
+3
-6
@@ -702,6 +702,7 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
702
"mrope embedding type requires multi-modal input mapper "
703
"returns 'image_grid_thw' or 'video_grid_thw'.")
704
705
+ second_per_grid_ts = mm_kwargs.get("second_per_grid_ts", None)
706
hf_config = self.runner.model_config.hf_config
707
708
inter_data.mrope_input_positions = [None] * inter_data.n_seqs
@@ -713,14 +714,10 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
714
mrope_input_positions, mrope_position_delta = \
715
MRotaryEmbedding.get_input_positions(
716
token_ids,
717
+ hf_config=hf_config,
718
image_grid_thw=image_grid_thw,
719
video_grid_thw=video_grid_thw,
718
- image_token_id=hf_config.image_token_id,
719
- video_token_id=hf_config.video_token_id,
720
- vision_start_token_id=hf_config.vision_start_token_id,
721
- vision_end_token_id=hf_config.vision_end_token_id,
722
- spatial_merge_size=hf_config.vision_config.
723
- spatial_merge_size,
720
+ second_per_grid_ts=second_per_grid_ts,
721
context_len=inter_data.context_lens[seq_idx],
722
seq_len=inter_data.seq_lens[seq_idx],
723
)