A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Model][VLM] Add Qwen2-VL model support (#7905)
Co-authored-by: Roger Wang <136131678+ywang96@users.noreply.github.com> Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>
Yang Fan committed
Sep 12, 2024 at 00:31 UTC
3b7fea770f44369d077e40010bb4983ff3641535
14 files changed
+1531
-31
docs/source/models/supported_models.rst
+7
-3
@@ -252,6 +252,11 @@ Multimodal Language Models
252
- Image\ :sup:`E`
253
- :code:`Qwen/Qwen-VL`, :code:`Qwen/Qwen-VL-Chat`, etc.
254
-
255
+ * - :code:`Qwen2VLForConditionalGeneration`
256
+ - Qwen2-VL (see note)
257
+ - Image\ :sup:`+` / Video\ :sup:`+`
258
+ - :code:`Qwen/Qwen2-VL-2B-Instruct`, :code:`Qwen/Qwen2-VL-7B-Instruct`, :code:`Qwen/Qwen2-VL-72B-Instruct`, etc.
259
+ -
260
* - :code:`UltravoxModel`
261
- Ultravox
262
- Audio\ :sup:`E+`
@@ -265,15 +270,14 @@ Multimodal Language Models
270
For :code:`openbmb/MiniCPM-V-2`, the official repo doesn't work yet, so we need to use a fork (:code:`HwwwH/MiniCPM-V-2`) for now.
271
For more details, please see: https://github.com/vllm-project/vllm/pull/4087#issuecomment-2250397630
272
268
- For :code:`LLaVA-NeXT-Video`, the latest release of :code:`huggingface/transformers` doesn't work yet, so we need to use a developer version (:code:`21fac7abba2a37fae86106f87fcf9974fd1e3830`) for now.
273
+.. note::
274
+ For :code:`LLaVA-NeXT-Video` and :code:`Qwen2-VL`, the latest release of :code:`huggingface/transformers` doesn't work yet, so we need to use a developer version (:code:`21fac7abba2a37fae86106f87fcf9974fd1e3830`) for now.
275
This can be installed by running the following command:
276
271
-
277
.. code-block:: bash
278
279
pip install git+https://github.com/huggingface/transformers.git@21fac7abba2a37fae86106f87fcf9974fd1e3830
280
276
-
281
----
282
283
If your model uses one of the above model architectures, you can seamlessly run your model with vLLM.
examples/offline_inference_vision_language.py
+18
@@ -179,6 +179,23 @@ def run_qwen_vl(question):
179
return llm, prompt, stop_token_ids
180
181
182
+# Qwen2-VL
183
+def run_qwen2_vl(question):
184
+ model_name = "Qwen/Qwen2-VL-7B-Instruct"
185
+
186
+ llm = LLM(
187
+ model=model_name,
188
+ max_num_seqs=5,
189
+ )
190
+
191
+ prompt = ("<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
192
+ "<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>"
193
+ f"{question}<|im_end|>\n"
194
+ "<|im_start|>assistant\n")
195
+ stop_token_ids = None
196
+ return llm, prompt, stop_token_ids
197
+
198
+
199
model_example_map = {
200
"llava": run_llava,
201
"llava-next": run_llava_next,
@@ -191,6 +208,7 @@ model_example_map = {
208
"blip-2": run_blip2,
209
"internvl_chat": run_internvl,
210
"qwen_vl": run_qwen_vl,
211
+ "qwen2_vl": run_qwen2_vl,
212
}
213
214
examples/offline_inference_vision_language_multi_image.py
+61
-7
@@ -6,7 +6,7 @@ by the model.
6
from argparse import Namespace
7
from typing import List
8
9
-from transformers import AutoTokenizer
9
+from transformers import AutoProcessor, AutoTokenizer
10
11
from vllm import LLM, SamplingParams
12
from vllm.multimodal.utils import fetch_image
@@ -30,7 +30,7 @@ def load_phi3v(question, image_urls: List[str]):
30
for i, _ in enumerate(image_urls, start=1))
31
prompt = f"<|user|>\n{placeholders}\n{question}<|end|>\n<|assistant|>\n"
32
stop_token_ids = None
33
- return llm, prompt, stop_token_ids
33
+ return llm, prompt, stop_token_ids, None
34
35
36
def load_internvl(question, image_urls: List[str]):
@@ -60,18 +60,72 @@ def load_internvl(question, image_urls: List[str]):
60
# https://huggingface.co/OpenGVLab/InternVL2-2B#service
61
stop_tokens = ["<|endoftext|>", "<|im_start|>", "<|im_end|>", "<|end|>"]
62
stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
63
- return llm, prompt, stop_token_ids
63
+
64
+ return llm, prompt, stop_token_ids, None
65
+
66
+
67
+def load_qwen2_vl(question, image_urls: List[str]):
68
+ try:
69
+ from qwen_vl_utils import process_vision_info
70
+ except ModuleNotFoundError:
71
+ print('WARNING: `qwen-vl-utils` not installed, input images will not '
72
+ 'be automatically resized. You can enable this functionality by '
73
+ '`pip install qwen-vl-utils`.')
74
+ process_vision_info = None
75
+
76
+ model_name = "Qwen/Qwen2-VL-7B-Instruct"
77
+
78
+ llm = LLM(
79
+ model=model_name,
80
+ max_num_seqs=5,
81
+ max_model_len=32768 if process_vision_info is None else 4096,
82
+ limit_mm_per_prompt={"image": len(image_urls)},
83
+ )
84
+
85
+ placeholders = [{"type": "image", "image": url} for url in image_urls]
86
+ messages = [{
87
+ "role": "system",
88
+ "content": "You are a helpful assistant."
89
+ }, {
90
+ "role":
91
+ "user",
92
+ "content": [
93
+ *placeholders,
94
+ {
95
+ "type": "text",
96
+ "text": question
97
+ },
98
+ ],
99
+ }]
100
+
101
+ processor = AutoProcessor.from_pretrained(model_name)
102
+
103
+ prompt = processor.apply_chat_template(messages,
104
+ tokenize=False,
105
+ add_generation_prompt=True)
106
+
107
+ stop_token_ids = None
108
+
109
+ if process_vision_info is None:
110
+ image_data = [fetch_image(url) for url in image_urls]
111
+ else:
112
+ image_data, _ = process_vision_info(messages)
113
+
114
+ return llm, prompt, stop_token_ids, image_data
115
116
117
model_example_map = {
118
"phi3_v": load_phi3v,
119
"internvl_chat": load_internvl,
120
+ "qwen2_vl": load_qwen2_vl,
121
}
122
123
124
def run_generate(model, question: str, image_urls: List[str]):
73
- llm, prompt, stop_token_ids = model_example_map[model](question,
74
- image_urls)
125
+ llm, prompt, stop_token_ids, image_data = model_example_map[model](
126
+ question, image_urls)
127
+ if image_data is None:
128
+ image_data = [fetch_image(url) for url in image_urls]
129
130
sampling_params = SamplingParams(temperature=0.0,
131
max_tokens=128,
@@ -81,7 +135,7 @@ def run_generate(model, question: str, image_urls: List[str]):
135
{
136
"prompt": prompt,
137
"multi_modal_data": {
84
- "image": [fetch_image(url) for url in image_urls]
138
+ "image": image_data
139
},
140
},
141
sampling_params=sampling_params)
@@ -92,7 +146,7 @@ def run_generate(model, question: str, image_urls: List[str]):
146
147
148
def run_chat(model: str, question: str, image_urls: List[str]):
95
- llm, _, stop_token_ids = model_example_map[model](question, image_urls)
149
+ llm, _, stop_token_ids, _ = model_example_map[model](question, image_urls)
150
151
sampling_params = SamplingParams(temperature=0.0,
152
max_tokens=128,
requirements-common.txt
+1
@@ -28,3 +28,4 @@ importlib_metadata
28
mistral_common >= 1.3.4
29
pyyaml
30
six>=1.16.0; python_version > '3.11' # transitive dependency of pandas that needs to be the latest version for python 3.12
31
+einops # Required for Qwen2-VL.
tests/models/test_registry.py
+5
@@ -1,9 +1,14 @@
1
import pytest
2
+import transformers
3
4
from vllm.model_executor.models import _MODELS, ModelRegistry
5
6
7
@pytest.mark.parametrize("model_cls", _MODELS)
8
def test_registry_imports(model_cls):
9
+ if (model_cls == "Qwen2VLForConditionalGeneration"
10
+ and transformers.__version__ < "4.45"):
11
+ pytest.skip("Waiting for next transformers release")
12
+
13
# Ensure all model classes can be imported successfully
14
ModelRegistry.resolve_model_cls([model_cls])
vllm/config.py
+6
-3
@@ -773,7 +773,7 @@ class LoadConfig:
773
ignore_patterns: The list of patterns to ignore when loading the model.
774
Default to "original/**/*" to avoid repeated loading of llama's
775
checkpoints.
776
-
776
+
777
"""
778
779
load_format: Union[str, LoadFormat, "BaseModelLoader"] = LoadFormat.AUTO
@@ -1733,8 +1733,11 @@ def _get_and_verify_max_len(
1733
"with rope_scaling. Please raise an issue so we can "
1734
"investigate.")
1735
1736
- assert "factor" in rope_scaling
1737
- scaling_factor = rope_scaling["factor"]
1736
+ if rope_type == "mrope":
1737
+ scaling_factor = 1
1738
+ else:
1739
+ assert "factor" in rope_scaling
1740
+ scaling_factor = rope_scaling["factor"]
1741
if rope_type == "yarn":
1742
derived_max_model_len = rope_scaling[
1743
"original_max_position_embeddings"]
vllm/entrypoints/chat_utils.py
+7
-1
@@ -108,7 +108,7 @@ class ConversationMessage(TypedDict, total=False):
108
"""The tool calls generated by the model, such as function calls."""
109
110
111
-ModalityStr = Literal["image", "audio"]
111
+ModalityStr = Literal["image", "audio", "video"]
112
_T = TypeVar("_T")
113
114
@@ -158,12 +158,18 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
158
hf_config.image_token_index)
159
if model_type in ("chameleon", "internvl_chat"):
160
return "<image>"
161
+ if model_type == "qwen2_vl":
162
+ return "<|vision_start|><|image_pad|><|vision_end|>"
163
164
raise TypeError(f"Unknown model type: {model_type}")
165
elif modality == "audio":
166
if model_type == "ultravox":
167
return "<|reserved_special_token_0|>"
168
raise TypeError(f"Unknown model type: {model_type}")
169
+ elif modality == "video":
170
+ if model_type == "qwen2_vl":
171
+ return "<|vision_start|><|video_pad|><|vision_end|>"
172
+ raise TypeError(f"Unknown model type: {model_type}")
173
else:
174
raise TypeError(f"Unknown modality: {modality}")
175
vllm/model_executor/layers/rotary_embedding.py
+184
-1
@@ -712,6 +712,179 @@ class Llama3RotaryEmbedding(RotaryEmbedding):
712
return new_freqs
713
714
715
+class MRotaryEmbedding(RotaryEmbedding):
716
+ """Rotary Embedding with Multimodal Sections."""
717
+
718
+ def __init__(
719
+ self,
720
+ head_size: int,
721
+ rotary_dim: int,
722
+ max_position_embeddings: int,
723
+ base: int,
724
+ is_neox_style: bool,
725
+ dtype: torch.dtype,
726
+ mrope_section: Optional[List[int]] = None,
727
+ ) -> None:
728
+ super().__init__(head_size, rotary_dim, max_position_embeddings, base,
729
+ is_neox_style, dtype)
730
+
731
+ self.mrope_section = mrope_section
732
+ if self.mrope_section:
733
+ assert sum(self.mrope_section) == rotary_dim // 2
734
+
735
+ def forward(
736
+ self,
737
+ positions: torch.Tensor,
738
+ query: torch.Tensor,
739
+ key: torch.Tensor,
740
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
741
+ """PyTorch-native implementation equivalent to forward().
742
+
743
+ Args:
744
+ positions:
745
+ [num_tokens,] (text only) or
746
+ [3, num_tokens] (T/H/W positions with multimodal inputs)
747
+ query: [num_tokens, num_heads * head_size]
748
+ key: [num_tokens, num_kv_heads * head_size]
749
+ """
750
+ assert positions.ndim == 1 or positions.ndim == 2
751
+
752
+ num_tokens = positions.shape[-1]
753
+ cos_sin = self.cos_sin_cache[positions]
754
+ cos, sin = cos_sin.chunk(2, dim=-1)
755
+ if positions.ndim == 2:
756
+ assert self.mrope_section
757
+
758
+ cos = torch.cat([
759
+ m[i]
760
+ for i, m in enumerate(cos.split(self.mrope_section, dim=-1))
761
+ ],
762
+ dim=-1)
763
+ sin = torch.cat([
764
+ m[i]
765
+ for i, m in enumerate(sin.split(self.mrope_section, dim=-1))
766
+ ],
767
+ dim=-1)
768
+
769
+ query_shape = query.shape
770
+ query = query.view(num_tokens, -1, self.head_size)
771
+ query_rot = query[..., :self.rotary_dim]
772
+ query_pass = query[..., self.rotary_dim:]
773
+ query_rot = _apply_rotary_emb(query_rot, cos, sin, self.is_neox_style)
774
+ query = torch.cat((query_rot, query_pass), dim=-1).reshape(query_shape)
775
+
776
+ key_shape = key.shape
777
+ key = key.view(num_tokens, -1, self.head_size)
778
+ key_rot = key[..., :self.rotary_dim]
779
+ key_pass = key[..., self.rotary_dim:]
780
+ key_rot = _apply_rotary_emb(key_rot, cos, sin, self.is_neox_style)
781
+ key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape)
782
+ return query, key
783
+
784
+ @staticmethod
785
+ def get_input_positions(
786
+ input_tokens: List[int],
787
+ image_grid_thw: Union[List[List[int]], torch.Tensor],
788
+ video_grid_thw: Union[List[List[int]], torch.Tensor],
789
+ image_token_id: int,
790
+ video_token_id: int,
791
+ vision_start_token_id: int,
792
+ vision_end_token_id: int,
793
+ spatial_merge_size: int,
794
+ context_len: int = 0,
795
+ ) -> Tuple[List[List[int]], int]:
796
+ """Get mrope input positions and delta value."""
797
+
798
+ if isinstance(image_grid_thw, torch.Tensor):
799
+ image_grid_thw = image_grid_thw.tolist()
800
+ if isinstance(video_grid_thw, torch.Tensor):
801
+ video_grid_thw = video_grid_thw.tolist()
802
+
803
+ input_tokens_tensor = torch.tensor(input_tokens)
804
+ vision_start_indices = torch.argwhere(
805
+ input_tokens_tensor == vision_start_token_id).squeeze(1)
806
+ vision_tokens = input_tokens_tensor[vision_start_indices + 1]
807
+ image_nums = (vision_tokens == image_token_id).sum()
808
+ video_nums = (vision_tokens == video_token_id).sum()
809
+ llm_pos_ids_list: list = []
810
+
811
+ st = 0
812
+ remain_images, remain_videos = image_nums, video_nums
813
+
814
+ image_index, video_index = 0, 0
815
+ for _ in range(image_nums + video_nums):
816
+ if image_token_id in input_tokens and remain_images > 0:
817
+ ed_image = input_tokens.index(image_token_id, st)
818
+ else:
819
+ ed_image = len(input_tokens) + 1
820
+ if video_token_id in input_tokens and remain_videos > 0:
821
+ ed_video = input_tokens.index(video_token_id, st)
822
+ else:
823
+ ed_video = len(input_tokens) + 1
824
+ if ed_image < ed_video:
825
+ t, h, w = (
826
+ image_grid_thw[image_index][0],
827
+ image_grid_thw[image_index][1],
828
+ image_grid_thw[image_index][2],
829
+ )
830
+ image_index += 1
831
+ remain_images -= 1
832
+ ed = ed_image
833
+ else:
834
+ t, h, w = (
835
+ video_grid_thw[video_index][0],
836
+ video_grid_thw[video_index][1],
837
+ video_grid_thw[video_index][2],
838
+ )
839
+ video_index += 1
840
+ remain_videos -= 1
841
+ ed = ed_video
842
+ llm_grid_t, llm_grid_h, llm_grid_w = \
843
+ t, h // spatial_merge_size, w // spatial_merge_size
844
+ text_len = ed - st
845
+
846
+ st_idx = llm_pos_ids_list[-1].max() + 1 if len(
847
+ llm_pos_ids_list) > 0 else 0
848
+ llm_pos_ids_list.append(
849
+ torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx)
850
+
851
+ t_index = torch.arange(llm_grid_t).view(-1, 1).expand(
852
+ -1, llm_grid_h * llm_grid_w).flatten()
853
+ h_index = torch.arange(llm_grid_h).view(1, -1, 1).expand(
854
+ llm_grid_t, -1, llm_grid_w).flatten()
855
+ w_index = torch.arange(llm_grid_w).view(1, 1, -1).expand(
856
+ llm_grid_t, llm_grid_h, -1).flatten()
857
+ llm_pos_ids_list.append(
858
+ torch.stack([t_index, h_index, w_index]) + text_len + st_idx)
859
+ st = ed + llm_grid_t * llm_grid_h * llm_grid_w
860
+
861
+ if st < len(input_tokens):
862
+ st_idx = llm_pos_ids_list[-1].max() + 1 if len(
863
+ llm_pos_ids_list) > 0 else 0
864
+ text_len = len(input_tokens) - st
865
+ llm_pos_ids_list.append(
866
+ torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx)
867
+
868
+ llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1)
869
+ llm_positions = llm_positions[:, context_len:]
870
+ mrope_position_delta = (llm_positions.max() + 1 -
871
+ len(input_tokens)).item()
872
+
873
+ return llm_positions.tolist(), mrope_position_delta
874
+
875
+ @staticmethod
876
+ def get_next_input_positions(
877
+ mrope_position_delta: int,
878
+ context_len: int,
879
+ seq_len: int,
880
+ ) -> List[List[int]]:
881
+ return [
882
+ list(
883
+ range(context_len + mrope_position_delta,
884
+ seq_len + mrope_position_delta)) for _ in range(3)
885
+ ]
886
+
887
+
888
_ROPE_DICT: Dict[Tuple, RotaryEmbedding] = {}
889
890
@@ -752,7 +925,7 @@ def get_rope(
925
# The correct one should be "longrope" but keep "su" here
926
# for backward compatible
927
if scaling_type not in {"su", "longrope"}:
755
- scaling_factor = rope_scaling["factor"]
928
+ scaling_factor = rope_scaling.get("factor", 1.0)
929
if scaling_type == "llama3":
930
low_freq_factor = rope_scaling["low_freq_factor"]
931
high_freq_factor = rope_scaling["high_freq_factor"]
@@ -816,6 +989,16 @@ def get_rope(
989
head_size, rotary_dim, max_position, original_max_position,
990
base, is_neox_style, dtype, short_factor, long_factor,
991
**extra_kwargs)
992
+ elif scaling_type == "mrope":
993
+ return MRotaryEmbedding(
994
+ head_size,
995
+ rotary_dim,
996
+ max_position,
997
+ base,
998
+ is_neox_style,
999
+ dtype,
1000
+ mrope_section=rope_scaling["mrope_section"],
1001
+ )
1002
else:
1003
raise ValueError(f"Unknown RoPE scaling type {scaling_type}")
1004
_ROPE_DICT[key] = rotary_emb
vllm/model_executor/models/__init__.py
+4
@@ -53,6 +53,8 @@ _GENERATION_MODELS = {
53
"PhiMoEForCausalLM": ("phimoe", "PhiMoEForCausalLM"),
54
"Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
55
"Qwen2MoeForCausalLM": ("qwen2_moe", "Qwen2MoeForCausalLM"),
56
+ "Qwen2VLForConditionalGeneration":
57
+ ("qwen2_vl", "Qwen2VLForConditionalGeneration"),
58
"RWForCausalLM": ("falcon", "FalconForCausalLM"),
59
"StableLMEpochForCausalLM": ("stablelm", "StablelmForCausalLM"),
60
"StableLmForCausalLM": ("stablelm", "StablelmForCausalLM"),
@@ -90,6 +92,8 @@ _MULTIMODAL_MODELS = {
92
"Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
93
"UltravoxModel": ("ultravox", "UltravoxModel"),
94
"QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
95
+ "Qwen2VLForConditionalGeneration": ("qwen2_vl",
96
+ "Qwen2VLForConditionalGeneration"),
97
}
98
_CONDITIONAL_GENERATION_MODELS = {
99
"BartModel": ("bart", "BartForConditionalGeneration"),
vllm/model_executor/models/qwen2_vl.py
new
+1088
@@ -0,0 +1,1088 @@
1
+# coding=utf-8
2
+# Adapted from
3
+# https://github.com/huggingface/transformers/blob/19e6e80e10118f855137b90740936c0b11ac397f/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py
4
+# Copyright 2024 The Qwen team.
5
+# Copyright 2023 The vLLM team.
6
+# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
7
+#
8
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
9
+# and OPT implementations in this library. It has been modified from its
10
+# original forms to accommodate minor architectural differences compared
11
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
12
+#
13
+# Licensed under the Apache License, Version 2.0 (the "License");
14
+# you may not use this file except in compliance with the License.
15
+# You may obtain a copy of the License at
16
+#
17
+# http://www.apache.org/licenses/LICENSE-2.0
18
+#
19
+# Unless required by applicable law or agreed to in writing, software
20
+# distributed under the License is distributed on an "AS IS" BASIS,
21
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
22
+# See the License for the specific language governing permissions and
23
+# limitations under the License.
24
+"""Inference-only Qwen2-VL model compatible with HuggingFace weights."""
25
+from array import array
26
+from functools import lru_cache, partial
27
+from typing import (Iterable, List, Mapping, Optional, Tuple, Type, TypedDict,
28
+ Union)
29
+
30
+import torch
31
+import torch.nn as nn
32
+import torch.nn.functional as F
33
+from einops import rearrange, repeat
34
+from PIL import Image
35
+from transformers import Qwen2VLConfig
36
+from transformers.image_utils import (get_image_size,
37
+ infer_channel_dimension_format,
38
+ to_numpy_array)
39
+from transformers.models.qwen2_vl.configuration_qwen2_vl import (
40
+ Qwen2VLVisionConfig)
41
+from transformers.models.qwen2_vl.image_processing_qwen2_vl import (
42
+ make_batched_images, make_batched_videos, smart_resize)
43
+
44
+import vllm.envs as envs
45
+from vllm.attention import AttentionMetadata
46
+from vllm.attention.selector import (_Backend, backend_name_to_enum,
47
+ get_global_forced_attn_backend)
48
+from vllm.config import CacheConfig, MultiModalConfig
49
+from vllm.distributed import parallel_state
50
+from vllm.distributed import utils as dist_utils
51
+from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
52
+from vllm.logger import init_logger
53
+from vllm.model_executor import SamplingMetadata
54
+from vllm.model_executor.layers.activation import QuickGELU
55
+from vllm.model_executor.layers.linear import (ColumnParallelLinear,
56
+ RowParallelLinear)
57
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
58
+from vllm.model_executor.layers.quantization import QuantizationConfig
59
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
60
+from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
61
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
62
+from vllm.model_executor.models.interfaces import SupportsMultiModal
63
+from vllm.model_executor.models.qwen2 import Qwen2Model
64
+from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalDataDict,
65
+ MultiModalInputs)
66
+from vllm.multimodal.base import MultiModalData
67
+from vllm.multimodal.image import cached_get_image_processor
68
+from vllm.platforms import current_platform
69
+from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
70
+ SequenceData)
71
+from vllm.transformers_utils.processor import get_processor
72
+
73
+logger = init_logger(__name__)
74
+
75
+# === Vision Inputs === #
76
+
77
+
78
+class Qwen2VLImageInputs(TypedDict):
79
+ pixel_values: torch.Tensor
80
+ """Shape:
81
+ `(num_patches, num_channels * patch_size * patch_size)`
82
+ """
83
+
84
+ image_grid_thw: torch.Tensor
85
+ """Shape: `(num_images, 3)`
86
+
87
+ This should be in `(grid_t, grid_h, grid_w)` format.
88
+ """
89
+
90
+
91
+class Qwen2VLVideoInputs(TypedDict):
92
+ pixel_values_videos: torch.Tensor
93
+ """Shape:
94
+ `(num_patches,
95
+ num_channels * temporal_patch_size * patch_size * patch_size)`
96
+ """
97
+
98
+ video_grid_thw: torch.Tensor
99
+ """Shape: `(num_videos, 3)`
100
+
101
+ This should be in `(grid_t, grid_h, grid_w)` format.
102
+ """
103
+
104
+
105
+# === Vision Encoder === #
106
+
107
+
108
+class Qwen2VisionMLP(nn.Module):
109
+
110
+ def __init__(
111
+ self,
112
+ in_features: int,
113
+ hidden_features: int = None,
114
+ act_layer: Type[nn.Module] = QuickGELU,
115
+ quant_config: Optional[QuantizationConfig] = None,
116
+ ):
117
+ super().__init__()
118
+ self.fc1 = ColumnParallelLinear(in_features,
119
+ hidden_features,
120
+ quant_config=quant_config)
121
+ self.act = act_layer()
122
+ self.fc2 = RowParallelLinear(hidden_features,
123
+ in_features,
124
+ quant_config=quant_config)
125
+
126
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
127
+ x_parallel, _ = self.fc1(x)
128
+ x_parallel = self.act(x_parallel)
129
+ x, _ = self.fc2(x_parallel)
130
+ return x
131
+
132
+
133
+def rotate_half(x: torch.Tensor, interleaved: bool = False) -> torch.Tensor:
134
+ if not interleaved:
135
+ x1, x2 = x.chunk(2, dim=-1)
136
+ return torch.cat((-x2, x1), dim=-1)
137
+ else:
138
+ x1, x2 = x[..., ::2], x[..., 1::2]
139
+ return rearrange(torch.stack((-x2, x1), dim=-1),
140
+ "... d two -> ... (d two)",
141
+ two=2)
142
+
143
+
144
+def apply_rotary_emb_torch(x: torch.Tensor,
145
+ cos: torch.Tensor,
146
+ sin: torch.Tensor,
147
+ interleaved: bool = False) -> torch.Tensor:
148
+ """
149
+ x: (batch_size, seqlen, nheads, headdim)
150
+ cos, sin: (seqlen, rotary_dim / 2) or (batch_size, seqlen, rotary_dim / 2)
151
+ """
152
+ ro_dim = cos.shape[-1] * 2
153
+ assert ro_dim <= x.shape[-1]
154
+ cos = repeat(
155
+ cos,
156
+ "... d -> ... 1 (2 d)" if not interleaved else "... d -> ... 1 (d 2)")
157
+ sin = repeat(
158
+ sin,
159
+ "... d -> ... 1 (2 d)" if not interleaved else "... d -> ... 1 (d 2)")
160
+ return torch.cat(
161
+ [
162
+ x[..., :ro_dim] * cos +
163
+ rotate_half(x[..., :ro_dim], interleaved) * sin, x[..., ro_dim:]
164
+ ],
165
+ dim=-1,
166
+ )
167
+
168
+
169
+def apply_rotary_pos_emb_vision(t: torch.Tensor,
170
+ freqs: torch.Tensor) -> torch.Tensor:
171
+ t_ = t.float()
172
+ cos = freqs.cos()
173
+ sin = freqs.sin()
174
+ output = apply_rotary_emb_torch(t_, cos, sin).type_as(t)
175
+ return output
176
+
177
+
178
+class Qwen2VisionAttention(nn.Module):
179
+
180
+ def __init__(
181
+ self,
182
+ embed_dim: Optional[int] = None,
183
+ num_heads: Optional[int] = None,
184
+ projection_size: Optional[int] = None,
185
+ quant_config: Optional[QuantizationConfig] = None,
186
+ ) -> None:
187
+ super().__init__()
188
+ # Per attention head and per partition values.
189
+ world_size = parallel_state.get_tensor_model_parallel_world_size()
190
+ self.hidden_size_per_attention_head = dist_utils.divide(
191
+ projection_size, num_heads)
192
+ self.num_attention_heads_per_partition = dist_utils.divide(
193
+ num_heads, world_size)
194
+
195
+ self.qkv = ColumnParallelLinear(input_size=embed_dim,
196
+ output_size=3 * projection_size,
197
+ quant_config=quant_config)
198
+ self.proj = RowParallelLinear(input_size=projection_size,
199
+ output_size=embed_dim,
200
+ quant_config=quant_config)
201
+
202
+ # Detect attention implementation.
203
+ selected_backend: Optional[_Backend] = get_global_forced_attn_backend()
204
+ if selected_backend is None:
205
+ backend_by_env_var: Optional[str] = envs.VLLM_ATTENTION_BACKEND
206
+ if backend_by_env_var is not None:
207
+ selected_backend = backend_name_to_enum(backend_by_env_var)
208
+ if selected_backend is None:
209
+ # For Volta and Turing GPUs, use xformers instead.
210
+ device_available = current_platform.get_device_capability()[0] >= 8
211
+ if device_available:
212
+ from transformers.utils import is_flash_attn_2_available
213
+
214
+ if is_flash_attn_2_available():
215
+ self._use_flash_attn = True
216
+ else:
217
+ logger.warning(
218
+ "Current Qwen2-VL implementation has a bug with "
219
+ "`vllm-flash-attn` inside vision module, so we use "
220
+ "xformers backend instead. You can run `pip install "
221
+ "flash-attn to use flash-attention backend.")
222
+ self._use_flash_attn = False
223
+ else:
224
+ self._use_flash_attn = False
225
+ else:
226
+ if selected_backend == _Backend.FLASH_ATTN:
227
+ self._use_flash_attn = True
228
+ elif selected_backend == _Backend.XFORMERS:
229
+ self._use_flash_attn = False
230
+ else:
231
+ raise RuntimeError(
232
+ f"Qwen2-VL does not support {selected_backend} backend now."
233
+ )
234
+
235
+ def forward(
236
+ self,
237
+ x: torch.Tensor,
238
+ cu_seqlens: torch.Tensor,
239
+ rotary_pos_emb: torch.Tensor = None,
240
+ ) -> torch.Tensor:
241
+ # [s, b, c] --> [s, b, head * 3 * head_dim]
242
+ x, _ = self.qkv(x)
243
+
244
+ # [s, b, head * 3 * head_dim] --> [s, b, head, 3 * head_dim]
245
+ new_x_shape = x.size()[:-1] + (
246
+ self.num_attention_heads_per_partition,
247
+ 3 * self.hidden_size_per_attention_head,
248
+ )
249
+ x = x.view(*new_x_shape)
250
+
251
+ # [s, b, head, 3 * head_dim] --> 3 [s, b, head, head_dim]
252
+ q, k, v = dist_utils.split_tensor_along_last_dim(x, 3)
253
+ batch_size = q.shape[1]
254
+
255
+ q, k, v = [
256
+ rearrange(x, "s b ... -> b s ...").contiguous() for x in (q, k, v)
257
+ ]
258
+ if rotary_pos_emb is not None:
259
+ q = apply_rotary_pos_emb_vision(q, rotary_pos_emb)
260
+ k = apply_rotary_pos_emb_vision(k, rotary_pos_emb)
261
+
262
+ if self._use_flash_attn:
263
+ # from vllm_flash_attn.flash_attn_interface import (
264
+ # flash_attn_varlen_func)
265
+ from flash_attn import flash_attn_varlen_func
266
+
267
+ q, k, v = [rearrange(x, "b s ... -> (b s) ...") for x in [q, k, v]]
268
+
269
+ max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max().item()
270
+ output = flash_attn_varlen_func(q,
271
+ k,
272
+ v,
273
+ cu_seqlens_q=cu_seqlens,
274
+ cu_seqlens_k=cu_seqlens,
275
+ max_seqlen_q=max_seqlen,
276
+ max_seqlen_k=max_seqlen,
277
+ dropout_p=0,
278
+ causal=False)
279
+
280
+ context_layer = rearrange(output,
281
+ "(b s) ... -> b s ...",
282
+ b=batch_size)
283
+ else:
284
+ from xformers import ops as xops
285
+ from xformers.ops.fmha.attn_bias import BlockDiagonalMask
286
+
287
+ seqlens = (cu_seqlens[1:] - cu_seqlens[:-1]).tolist()
288
+ attn_bias = BlockDiagonalMask.from_seqlens(q_seqlen=seqlens,
289
+ kv_seqlen=None)
290
+
291
+ context_layer = xops.memory_efficient_attention_forward(
292
+ q, k, v, attn_bias=attn_bias, p=0, scale=None)
293
+ context_layer = rearrange(context_layer,
294
+ "b s h d -> s b (h d)").contiguous()
295
+
296
+ output, _ = self.proj(context_layer)
297
+ return output
298
+
299
+
300
+class Qwen2VisionBlock(nn.Module):
301
+
302
+ def __init__(
303
+ self,
304
+ dim: int,
305
+ num_heads: int,
306
+ mlp_ratio: float,
307
+ act_layer: Type[nn.Module] = QuickGELU,
308
+ norm_layer: Type[nn.Module] = None,
309
+ quant_config: Optional[QuantizationConfig] = None,
310
+ ) -> None:
311
+ super().__init__()
312
+ if norm_layer is None:
313
+ norm_layer = partial(nn.LayerNorm, eps=1e-6)
314
+ self.norm1 = norm_layer(dim)
315
+ self.norm2 = norm_layer(dim)
316
+ mlp_hidden_dim = int(dim * mlp_ratio)
317
+
318
+ self.attn = Qwen2VisionAttention(embed_dim=dim,
319
+ num_heads=num_heads,
320
+ projection_size=dim,
321
+ quant_config=quant_config)
322
+ self.mlp = Qwen2VisionMLP(dim,
323
+ mlp_hidden_dim,
324
+ act_layer=act_layer,
325
+ quant_config=quant_config)
326
+
327
+ def forward(self, x: torch.Tensor, cu_seqlens: torch.Tensor,
328
+ rotary_pos_emb: torch.Tensor) -> torch.Tensor:
329
+ x = x + self.attn(self.norm1(x),
330
+ cu_seqlens=cu_seqlens,
331
+ rotary_pos_emb=rotary_pos_emb)
332
+ x = x + self.mlp(self.norm2(x))
333
+ return x
334
+
335
+
336
+class Qwen2VisionPatchEmbed(nn.Module):
337
+
338
+ def __init__(
339
+ self,
340
+ patch_size: int = 14,
341
+ temporal_patch_size: int = 2,
342
+ in_chans: int = 3,
343
+ embed_dim: int = 1152,
344
+ ) -> None:
345
+ super().__init__()
346
+ self.patch_size = patch_size
347
+ self.temporal_patch_size = temporal_patch_size
348
+ self.embed_dim = embed_dim
349
+
350
+ kernel_size = [temporal_patch_size, patch_size, patch_size]
351
+ self.proj = nn.Conv3d(in_chans,
352
+ embed_dim,
353
+ kernel_size=kernel_size,
354
+ stride=kernel_size,
355
+ bias=False)
356
+
357
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
358
+ L, C = x.shape
359
+ x = x.view(L, -1, self.temporal_patch_size, self.patch_size,
360
+ self.patch_size)
361
+ x = self.proj(x).view(L, self.embed_dim)
362
+ return x
363
+
364
+
365
+class Qwen2VisionPatchMerger(nn.Module):
366
+
367
+ def __init__(
368
+ self,
369
+ d_model: int,
370
+ context_dim: int,
371
+ norm_layer: Type[nn.Module] = None,
372
+ spatial_merge_size: int = 2,
373
+ quant_config: Optional[QuantizationConfig] = None,
374
+ ) -> None:
375
+ super().__init__()
376
+ self.hidden_size = context_dim * (spatial_merge_size**2)
377
+ if norm_layer is None:
378
+ norm_layer = partial(nn.LayerNorm, eps=1e-6)
379
+ self.ln_q = norm_layer(context_dim)
380
+ self.mlp = nn.ModuleList([
381
+ ColumnParallelLinear(self.hidden_size,
382
+ self.hidden_size,
383
+ bias=True,
384
+ quant_config=quant_config),
385
+ nn.GELU(),
386
+ RowParallelLinear(self.hidden_size,
387
+ d_model,
388
+ bias=True,
389
+ quant_config=quant_config),
390
+ ])
391
+
392
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
393
+ x = self.ln_q(x)
394
+ x = x.view(-1, self.hidden_size)
395
+
396
+ mlp_fc1, mlp_act, mlp_fc2 = self.mlp
397
+ x_parallel, _ = mlp_fc1(x)
398
+ x_parallel = mlp_act(x_parallel)
399
+ out, _ = mlp_fc2(x_parallel)
400
+ return out
401
+
402
+
403
+class Qwen2VisionRotaryEmbedding(nn.Module):
404
+
405
+ def __init__(self, dim: int, theta: float = 10000.0) -> None:
406
+ super().__init__()
407
+ self.dim = dim
408
+ self.theta = theta
409
+ inv_freq = 1.0 / (theta
410
+ **(torch.arange(0, dim, 2, dtype=torch.float) / dim))
411
+ self.register_buffer("inv_freq", inv_freq, persistent=False)
412
+ self._seq_len_cached = 0
413
+ self._freqs_cached = None
414
+
415
+ def update_freqs_cache(self, seqlen: int) -> None:
416
+ if seqlen > self._seq_len_cached:
417
+ seqlen *= 2
418
+ self._seq_len_cached = seqlen
419
+ self.inv_freq = 1.0 / (self.theta**(torch.arange(
420
+ 0, self.dim, 2, dtype=torch.float, device=self.inv_freq.device)
421
+ / self.dim))
422
+ seq = torch.arange(seqlen,
423
+ device=self.inv_freq.device,
424
+ dtype=self.inv_freq.dtype)
425
+ freqs = torch.outer(seq, self.inv_freq)
426
+ self._freqs_cached = freqs
427
+
428
+ def forward(self, seqlen: int) -> torch.Tensor:
429
+ self.update_freqs_cache(seqlen)
430
+ return self._freqs_cached[:seqlen]
431
+
432
+
433
+class Qwen2VisionTransformer(nn.Module):
434
+
435
+ def __init__(
436
+ self,
437
+ vision_config: Qwen2VLVisionConfig,
438
+ norm_eps: float = 1e-6,
439
+ quant_config: Optional[QuantizationConfig] = None,
440
+ ) -> None:
441
+ super().__init__()
442
+
443
+ patch_size: int = vision_config.patch_size
444
+ temporal_patch_size: int = vision_config.temporal_patch_size
445
+ spatial_merge_size: int = vision_config.spatial_merge_size
446
+ in_chans: int = vision_config.in_chans
447
+ hidden_size: int = vision_config.hidden_size
448
+ embed_dim: int = vision_config.embed_dim
449
+ depth: int = vision_config.depth
450
+ num_heads: int = vision_config.num_heads
451
+ mlp_ratio: float = vision_config.mlp_ratio
452
+
453
+ self.spatial_merge_size = spatial_merge_size
454
+
455
+ self.patch_embed = Qwen2VisionPatchEmbed(
456
+ patch_size=patch_size,
457
+ temporal_patch_size=temporal_patch_size,
458
+ in_chans=in_chans,
459
+ embed_dim=embed_dim,
460
+ )
461
+
462
+ norm_layer = partial(nn.LayerNorm, eps=norm_eps)
463
+ head_dim = embed_dim // num_heads
464
+ self.rotary_pos_emb = Qwen2VisionRotaryEmbedding(head_dim // 2)
465
+
466
+ self.blocks = nn.ModuleList([
467
+ Qwen2VisionBlock(
468
+ dim=embed_dim,
469
+ num_heads=num_heads,
470
+ mlp_ratio=mlp_ratio,
471
+ norm_layer=norm_layer,
472
+ quant_config=quant_config,
473
+ ) for _ in range(depth)
474
+ ])
475
+ self.merger = Qwen2VisionPatchMerger(
476
+ d_model=hidden_size,
477
+ context_dim=embed_dim,
478
+ norm_layer=norm_layer,
479
+ quant_config=quant_config,
480
+ )
481
+
482
+ @property
483
+ def dtype(self) -> torch.dtype:
484
+ return self.blocks[0].mlp.fc2.weight.dtype
485
+
486
+ @property
487
+ def device(self) -> torch.device:
488
+ return self.blocks[0].mlp.fc2.weight.device
489
+
490
+ def rot_pos_emb(self, grid_thw: torch.Tensor) -> torch.Tensor:
491
+ pos_ids = []
492
+ for t, h, w in grid_thw:
493
+ hpos_ids = torch.arange(h).unsqueeze(1).expand(-1, w)
494
+ wpos_ids = torch.arange(w).unsqueeze(0).expand(h, -1)
495
+ hpos_ids = hpos_ids.reshape(
496
+ h // self.spatial_merge_size,
497
+ self.spatial_merge_size,
498
+ w // self.spatial_merge_size,
499
+ self.spatial_merge_size,
500
+ ).permute(0, 2, 1, 3).flatten()
501
+ wpos_ids = wpos_ids.reshape(
502
+ h // self.spatial_merge_size,
503
+ self.spatial_merge_size,
504
+ w // self.spatial_merge_size,
505
+ self.spatial_merge_size,
506
+ ).permute(0, 2, 1, 3).flatten()
507
+ pos_ids.append(
508
+ torch.stack([hpos_ids, wpos_ids], dim=-1).repeat(t, 1))
509
+ pos_ids = torch.cat(pos_ids, dim=0)
510
+ max_grid_size = grid_thw[:, 1:].max()
511
+ rotary_pos_emb_full = self.rotary_pos_emb(max_grid_size)
512
+ rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1)
513
+ return rotary_pos_emb
514
+
515
+ def forward(
516
+ self,
517
+ x: torch.Tensor,
518
+ grid_thw: torch.Tensor,
519
+ ) -> torch.Tensor:
520
+ # patchify
521
+ x = x.to(device=self.device, dtype=self.dtype)
522
+ x = self.patch_embed(x)
523
+
524
+ # compute position embedding
525
+ rotary_pos_emb = self.rot_pos_emb(grid_thw)
526
+
527
+ # compute cu_seqlens
528
+ cu_seqlens = torch.repeat_interleave(grid_thw[:, 1] * grid_thw[:, 2],
529
+ grid_thw[:, 0]).cumsum(
530
+ dim=0, dtype=torch.int32)
531
+ cu_seqlens = F.pad(cu_seqlens, (1, 0), "constant", 0)
532
+
533
+ # transformers
534
+ x = x.unsqueeze(1)
535
+ for blk in self.blocks:
536
+ x = blk(x, cu_seqlens=cu_seqlens, rotary_pos_emb=rotary_pos_emb)
537
+
538
+ # adapter
539
+ x = self.merger(x)
540
+ return x
541
+
542
+
543
+# === Vision input helpers === #
544
+
545
+cached_get_processor = lru_cache(get_processor)
546
+
547
+
548
+def mm_input_mapper_for_qwen2_vl(
549
+ ctx: InputContext,
550
+ data: MultiModalData[object],
551
+ data_type_key: str,
552
+) -> MultiModalInputs:
553
+ """Input mapper for Qwen2-VL."""
554
+ model_config = ctx.model_config
555
+ image_processor = cached_get_image_processor(
556
+ model_config.model, trust_remote_code=model_config.trust_remote_code)
557
+ if image_processor is None:
558
+ raise RuntimeError("No HuggingFace processor is available "
559
+ "to process the image object")
560
+
561
+ images = None
562
+ videos = None
563
+ if data_type_key == "image":
564
+ images = data
565
+ else:
566
+ assert data_type_key == "video"
567
+ videos = data
568
+
569
+ try:
570
+ batch_data = image_processor \
571
+ .preprocess(images=images, videos=videos, return_tensors="pt") \
572
+ .data
573
+ except Exception:
574
+ logger.error("Failed to process image (%s)", data)
575
+ raise
576
+
577
+ return MultiModalInputs(batch_data)
578
+
579
+
580
+image_input_mapper_for_qwen2_vl = partial(mm_input_mapper_for_qwen2_vl,
581
+ data_type_key="image")
582
+video_input_mapper_for_qwen2_vl = partial(mm_input_mapper_for_qwen2_vl,
583
+ data_type_key="video")
584
+
585
+
586
+def _get_vision_info(
587
+ image_processor,
588
+ height: int,
589
+ width: int,
590
+ min_pixels: int,
591
+ max_pixels: int,
592
+ do_resize: bool = True,
593
+ data_type_key: str = "image",
594
+ mm_count: int = 1,
595
+):
596
+ """Get information (resized height / width and number of vision tokens)
597
+ of input image / video frame."""
598
+
599
+ if do_resize:
600
+ resized_height, resized_width = smart_resize(
601
+ height=height,
602
+ width=width,
603
+ factor=image_processor.patch_size * image_processor.merge_size,
604
+ min_pixels=min_pixels,
605
+ max_pixels=max_pixels,
606
+ )
607
+ else:
608
+ resized_height, resized_width = height, width
609
+
610
+ if data_type_key == "image":
611
+ grid_t = mm_count
612
+ else:
613
+ assert data_type_key == "video"
614
+ grid_t = max(mm_count // image_processor.temporal_patch_size, 1)
615
+
616
+ grid_h = resized_height // image_processor.patch_size
617
+ grid_w = resized_width // image_processor.patch_size
618
+ vision_tokens = grid_t * grid_h * grid_w
619
+ llm_num_vision_tokens = (vision_tokens // image_processor.merge_size //
620
+ image_processor.merge_size)
621
+
622
+ return resized_height, resized_width, llm_num_vision_tokens
623
+
624
+
625
+def _get_max_image_info(
626
+ image_processor,
627
+ data_type_key: str = "image",
628
+ mm_count: int = 1,
629
+):
630
+ return _get_vision_info(
631
+ image_processor,
632
+ height=9999999,
633
+ width=9999999,
634
+
635
+ # Limit min / max pixels.
636
+ min_pixels=max(image_processor.min_pixels, 28 * 28),
637
+ max_pixels=min(image_processor.max_pixels, 1280 * 28 * 28),
638
+ data_type_key=data_type_key,
639
+ mm_count=mm_count,
640
+ )
641
+
642
+
643
+def get_max_qwen2_vl_mm_tokens(ctx: InputContext, data_type_key: str) -> int:
644
+ image_processor = cached_get_image_processor(ctx.model_config.model)
645
+ max_resized_height, max_resized_width, max_llm_image_tokens = \
646
+ _get_max_image_info(image_processor, data_type_key=data_type_key,
647
+ mm_count=1)
648
+ return max_llm_image_tokens
649
+
650
+
651
+get_max_qwen2_vl_image_tokens = partial(get_max_qwen2_vl_mm_tokens,
652
+ data_type_key="image")
653
+get_max_qwen2_vl_video_tokens = partial(get_max_qwen2_vl_mm_tokens,
654
+ data_type_key="video")
655
+
656
+
657
+def dummy_data_for_qwen2_vl(
658
+ ctx: InputContext, seq_len: int, mm_counts: Mapping[str, int]
659
+) -> Tuple[SequenceData, Optional[MultiModalDataDict]]:
660
+ image_processor = cached_get_image_processor(ctx.model_config.model)
661
+
662
+ num_images = mm_counts["image"]
663
+ max_resized_height, max_resized_width, max_llm_image_tokens = \
664
+ _get_max_image_info(image_processor, data_type_key="image",
665
+ mm_count=num_images)
666
+ if seq_len - max_llm_image_tokens - 2 < 0:
667
+ raise RuntimeError(
668
+ f"Qwen2-VL cannot process {num_images} images in a prompt, "
669
+ "please increase max_model_len or reduce image limit by "
670
+ "--limit-mm-per-prompt.")
671
+
672
+ # Check video counts.
673
+ num_videos = mm_counts["video"]
674
+ max_resized_height, max_resized_width, max_llm_video_tokens = \
675
+ _get_max_image_info(image_processor, data_type_key="video",
676
+ mm_count=num_videos)
677
+ if seq_len - max_llm_video_tokens - 2 < 0:
678
+ raise RuntimeError(
679
+ f"Qwen2-VL cannot process {num_images} videos in a prompt, "
680
+ "please increase max_model_len or reduce video limit by "
681
+ "--limit-mm-per-prompt.")
682
+
683
+ hf_config = ctx.get_hf_config(Qwen2VLConfig)
684
+ token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE,
685
+ [hf_config.vision_start_token_id])
686
+ token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE,
687
+ [hf_config.image_token_id]) * max_llm_image_tokens
688
+ token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE,
689
+ [hf_config.vision_end_token_id])
690
+ token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE,
691
+ [0]) * (seq_len - max_llm_image_tokens - 2)
692
+ dummy_seqdata = SequenceData(token_ids)
693
+ dummy_image = Image.new("RGB", (max_resized_width, max_resized_height),
694
+ color=0)
695
+
696
+ return dummy_seqdata, {
697
+ "image": dummy_image if num_images == 1 else [dummy_image] * num_images
698
+ }
699
+
700
+
701
+def _get_llm_num_vision_tokens(
702
+ mm_inputs: list,
703
+ data_type_key: str,
704
+ image_processor,
705
+):
706
+ """Get number of vision tokens of multimodal inputs.
707
+
708
+ This method is derived from `transformers.models.qwen2_vl.
709
+ image_processing_qwen2_vl.Qwen2VLImageProcessor._preprocess`.
710
+ """
711
+ image = to_numpy_array(mm_inputs[0])
712
+ input_data_format = infer_channel_dimension_format(image)
713
+ height, width = get_image_size(image, channel_dim=input_data_format)
714
+ _, _, llm_num_vision_tokens = _get_vision_info(
715
+ image_processor,
716
+ height=height,
717
+ width=width,
718
+ min_pixels=image_processor.min_pixels,
719
+ max_pixels=image_processor.max_pixels,
720
+ do_resize=image_processor.do_resize,
721
+ data_type_key=data_type_key,
722
+ mm_count=len(mm_inputs),
723
+ )
724
+ return llm_num_vision_tokens
725
+
726
+
727
+def input_processor_for_qwen2_vl(ctx: InputContext,
728
+ llm_inputs: LLMInputs) -> LLMInputs:
729
+ multi_modal_data = llm_inputs.get("multi_modal_data", None)
730
+ if multi_modal_data is None:
731
+ return llm_inputs
732
+
733
+ image_inputs = multi_modal_data.get("image", None)
734
+ video_inputs = multi_modal_data.get("video", None)
735
+
736
+ processor = cached_get_processor(ctx.model_config.model)
737
+ image_processor = processor.image_processor
738
+ hf_config = ctx.get_hf_config(Qwen2VLConfig)
739
+
740
+ # To avoid redundant processing of vision objects (resize, rescale, etc.),
741
+ # we extract code of calculating number of vision tokens from
742
+ # `transformers.models.qwen2_vl.processing_qwen2_vl.Qwen2VLProcessor`.
743
+ #
744
+ # The following code is equivalent to:
745
+ # prompt = llm_inputs["prompt"]
746
+ # inputs = processor(text=[prompt],
747
+ # images=image_inputs,
748
+ # videos=video_inputs,
749
+ # padding=True,
750
+ # return_tensors="pt")
751
+ # prompt_token_ids = inputs["input_ids"][0].tolist()
752
+
753
+ prompt_token_ids = llm_inputs.get("prompt_token_ids", None)
754
+ if prompt_token_ids is None:
755
+ prompt = llm_inputs["prompt"]
756
+ prompt_token_ids = processor.tokenizer(
757
+ prompt,
758
+ padding=True,
759
+ return_tensors=None,
760
+ )["input_ids"]
761
+
762
+ # Expand image pad tokens.
763
+ if image_inputs is not None:
764
+ image_indices = [
765
+ idx for idx, token in enumerate(prompt_token_ids)
766
+ if token == hf_config.image_token_id
767
+ ]
768
+ image_inputs = make_batched_images(image_inputs)
769
+ assert len(image_indices) == len(image_inputs)
770
+
771
+ prompt_token_ids_with_image = []
772
+ for image_cnt, image in enumerate(image_inputs):
773
+ num_image_tokens = _get_llm_num_vision_tokens(
774
+ [image],
775
+ data_type_key="image",
776
+ image_processor=image_processor,
777
+ )
778
+ if image_cnt == 0:
779
+ non_image_tokens = prompt_token_ids[:image_indices[image_cnt]]
780
+ else:
781
+ non_image_tokens = prompt_token_ids[image_indices[image_cnt -
782
+ 1] +
783
+ 1:image_indices[image_cnt]]
784
+ prompt_token_ids_with_image.extend(non_image_tokens)
785
+ prompt_token_ids_with_image.extend(
786
+ hf_config.image_token_id for _ in range(num_image_tokens))
787
+ prompt_token_ids_with_image.extend(prompt_token_ids[image_indices[-1] +
788
+ 1:])
789
+ prompt_token_ids = prompt_token_ids_with_image
790
+
791
+ # Expand video pad tokens.
792
+ if video_inputs is not None:
793
+ video_indices = [
794
+ idx for idx, token in enumerate(prompt_token_ids)
795
+ if token == hf_config.video_token_id
796
+ ]
797
+ video_inputs = make_batched_videos(video_inputs)
798
+ assert len(video_indices) == len(video_inputs)
799
+
800
+ prompt_token_ids_with_video = []
801
+ for video_cnt, video in enumerate(video_inputs):
802
+ num_video_tokens = _get_llm_num_vision_tokens(
803
+ video,
804
+ data_type_key="video",
805
+ image_processor=image_processor,
806
+ )
807
+ if video_cnt == 0:
808
+ non_video_tokens = prompt_token_ids[:video_indices[video_cnt]]
809
+ else:
810
+ non_video_tokens = prompt_token_ids[video_indices[video_cnt -
811
+ 1] +
812
+ 1:video_indices[video_cnt]]
813
+ prompt_token_ids_with_video.extend(non_video_tokens)
814
+ prompt_token_ids_with_video.extend(
815
+ hf_config.video_token_id for _ in range(num_video_tokens))
816
+ prompt_token_ids_with_video.extend(prompt_token_ids[video_indices[-1] +
817
+ 1:])
818
+ prompt_token_ids = prompt_token_ids_with_video
819
+
820
+ return LLMInputs(
821
+ prompt_token_ids=prompt_token_ids,
822
+ prompt=llm_inputs["prompt"],
823
+ multi_modal_data=multi_modal_data,
824
+ )
825
+
826
+
827
+@MULTIMODAL_REGISTRY.register_image_input_mapper(
828
+ image_input_mapper_for_qwen2_vl)
829
+@MULTIMODAL_REGISTRY.register_input_mapper("video",
830
+ video_input_mapper_for_qwen2_vl)
831
+@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_qwen2_vl_image_tokens)
832
+@MULTIMODAL_REGISTRY.register_max_multimodal_tokens(
833
+ "video", get_max_qwen2_vl_video_tokens)
834
+@INPUT_REGISTRY.register_dummy_data(dummy_data_for_qwen2_vl)
835
+@INPUT_REGISTRY.register_input_processor(input_processor_for_qwen2_vl)
836
+class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal):
837
+
838
+ def __init__(self,
839
+ config: Qwen2VLConfig,
840
+ multimodal_config: MultiModalConfig,
841
+ cache_config: Optional[CacheConfig] = None,
842
+ quant_config: Optional[QuantizationConfig] = None) -> None:
843
+ super().__init__()
844
+
845
+ assert not cache_config.enable_prefix_caching, \
846
+ "Qwen2-VL currently does not support prefix caching"
847
+
848
+ self.config = config
849
+ self.multimodal_config = multimodal_config
850
+
851
+ self.visual = Qwen2VisionTransformer(
852
+ config.vision_config,
853
+ norm_eps=getattr(config, "rms_norm_eps", 1e-6),
854
+
855
+ # NOTE: Qwen2-VL vision encoder does not support any
856
+ # quantization method now.
857
+ quant_config=None,
858
+ )
859
+
860
+ self.model = Qwen2Model(config, cache_config, quant_config)
861
+
862
+ if config.tie_word_embeddings:
863
+ self.lm_head = self.model.embed_tokens
864
+ else:
865
+ self.lm_head = ParallelLMHead(config.vocab_size,
866
+ config.hidden_size,
867
+ quant_config=quant_config)
868
+
869
+ self.logits_processor = LogitsProcessor(config.vocab_size)
870
+ self.sampler = Sampler()
871
+
872
+ def _validate_and_reshape_mm_tensor(self,
873
+ mm_input: Union[torch.Tensor,
874
+ List[torch.Tensor]],
875
+ name: str) -> torch.Tensor:
876
+ if not isinstance(mm_input, (torch.Tensor, list)):
877
+ raise ValueError(f"Incorrect type of {name}. "
878
+ f"Got type: {type(mm_input)}")
879
+ if isinstance(mm_input, torch.Tensor):
880
+ if mm_input.ndim == 2:
881
+ return mm_input
882
+ if mm_input.ndim != 3:
883
+ raise ValueError(f"{name} should be 2D or batched 3D tensor. "
884
+ f"Got ndim: {mm_input.ndim}")
885
+ return torch.concat(list(mm_input))
886
+ else:
887
+ return torch.concat(mm_input)
888
+
889
+ def _parse_and_validate_image_input(
890
+ self, **kwargs: object) -> Optional[Qwen2VLImageInputs]:
891
+ pixel_values = kwargs.pop("pixel_values", None)
892
+ image_grid_thw = kwargs.pop("image_grid_thw", None)
893
+
894
+ if pixel_values is None:
895
+ return None
896
+
897
+ pixel_values = self._validate_and_reshape_mm_tensor(
898
+ pixel_values, "image pixel values")
899
+ image_grid_thw = self._validate_and_reshape_mm_tensor(
900
+ image_grid_thw, "image grid_thw")
901
+
902
+ if not isinstance(pixel_values, (torch.Tensor, list)):
903
+ raise ValueError("Incorrect type of image pixel values. "
904
+ f"Got type: {type(pixel_values)}")
905
+
906
+ return Qwen2VLImageInputs(pixel_values=pixel_values,
907
+ image_grid_thw=image_grid_thw)
908
+
909
+ def _parse_and_validate_video_input(
910
+ self, **kwargs: object) -> Optional[Qwen2VLVideoInputs]:
911
+ pixel_values_videos = kwargs.pop("pixel_values_videos", None)
912
+ video_grid_thw = kwargs.pop("video_grid_thw", None)
913
+
914
+ if pixel_values_videos is None:
915
+ return None
916
+
917
+ pixel_values_videos = self._validate_and_reshape_mm_tensor(
918
+ pixel_values_videos, "video pixel values")
919
+ video_grid_thw = self._validate_and_reshape_mm_tensor(
920
+ video_grid_thw, "video grid_thw")
921
+
922
+ return Qwen2VLVideoInputs(
923
+ pixel_values_videos=pixel_values_videos,
924
+ video_grid_thw=video_grid_thw,
925
+ )
926
+
927
+ def _process_image_input(self,
928
+ image_input: Qwen2VLImageInputs) -> torch.Tensor:
929
+ pixel_values = image_input["pixel_values"].type(self.visual.dtype)
930
+ image_embeds = self.visual(pixel_values,
931
+ grid_thw=image_input["image_grid_thw"])
932
+ return image_embeds
933
+
934
+ def _process_video_input(self,
935
+ video_input: Qwen2VLVideoInputs) -> torch.Tensor:
936
+ pixel_values_videos = video_input["pixel_values_videos"].type(
937
+ self.visual.dtype)
938
+ video_embeds = self.visual(pixel_values_videos,
939
+ grid_thw=video_input["video_grid_thw"])
940
+ return video_embeds
941
+
942
+ def _merge_multimodal_embeddings(
943
+ self,
944
+ input_ids: torch.Tensor,
945
+ inputs_embeds: torch.Tensor,
946
+ multimodal_embeddings: torch.Tensor,
947
+ placeholder_token_id: int,
948
+ ) -> torch.Tensor:
949
+ mask = (input_ids == placeholder_token_id)
950
+ inputs_embeds[mask, :] = multimodal_embeddings
951
+ return inputs_embeds
952
+
953
+ def forward(
954
+ self,
955
+ input_ids: torch.Tensor,
956
+ positions: torch.Tensor,
957
+ kv_caches: List[torch.Tensor],
958
+ attn_metadata: AttentionMetadata,
959
+ intermediate_tensors: Optional[IntermediateTensors] = None,
960
+ **kwargs: object,
961
+ ) -> SamplerOutput:
962
+ """Run forward pass for Qwen2-VL.
963
+
964
+ Args:
965
+ input_ids: Flattened (concatenated) input_ids corresponding to a
966
+ batch.
967
+ positions: Flattened (concatenated) position ids corresponding to a
968
+ batch.
969
+ **NOTE**: If mrope is enabled (default setting for Qwen2-VL
970
+ opensource models), the shape will be `(3, seq_len)`,
971
+ otherwise it will be `(seq_len,).
972
+ pixel_values: Pixel values to be fed to a model.
973
+ `None` if no images are passed.
974
+ image_grid_thw: Tensor `(n_images, 3)` of image 3D grid in LLM.
975
+ `None` if no images are passed.
976
+ pixel_values_videos: Pixel values of videos to be fed to a model.
977
+ `None` if no videos are passed.
978
+ video_grid_thw: Tensor `(n_videos, 3)` of video 3D grid in LLM.
979
+ `None` if no videos are passed.
980
+ """
981
+
982
+ image_input = self._parse_and_validate_image_input(**kwargs)
983
+ video_input = self._parse_and_validate_video_input(**kwargs)
984
+
985
+ if image_input is None and video_input is None:
986
+ inputs_embeds = None
987
+ else:
988
+ if getattr(self.config, "rope_scaling", {}).get("type",
989
+ None) == "mrope":
990
+ assert positions.ndim == 2 and positions.size(0) == 3, (
991
+ "multimodal section rotary embedding requires "
992
+ f"(3, seq_len) positions, but got {positions.size()}")
993
+
994
+ inputs_embeds = self.model.embed_tokens(input_ids)
995
+
996
+ if image_input is not None:
997
+ image_embeds = self._process_image_input(image_input)
998
+ inputs_embeds = self._merge_multimodal_embeddings(
999
+ input_ids,
1000
+ inputs_embeds,
1001
+ image_embeds,
1002
+ placeholder_token_id=self.config.image_token_id,
1003
+ )
1004
+
1005
+ if video_input is not None:
1006
+ video_embeds = self._process_video_input(video_input)
1007
+ inputs_embeds = self._merge_multimodal_embeddings(
1008
+ input_ids,
1009
+ inputs_embeds,
1010
+ video_embeds,
1011
+ placeholder_token_id=self.config.video_token_id,
1012
+ )
1013
+
1014
+ input_ids = None
1015
+
1016
+ hidden_states = self.model(
1017
+ input_ids=input_ids,
1018
+ positions=positions,
1019
+ kv_caches=kv_caches,
1020
+ attn_metadata=attn_metadata,
1021
+ inputs_embeds=inputs_embeds,
1022
+ )
1023
+ return hidden_states
1024
+
1025
+ def compute_logits(self, hidden_states: torch.Tensor,
1026
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
1027
+ logits = self.logits_processor(self.lm_head, hidden_states,
1028
+ sampling_metadata)
1029
+ return logits
1030
+
1031
+ def sample(
1032
+ self,
1033
+ logits: torch.Tensor,
1034
+ sampling_metadata: SamplingMetadata,
1035
+ ) -> Optional[SamplerOutput]:
1036
+ next_tokens = self.sampler(logits, sampling_metadata)
1037
+ return next_tokens
1038
+
1039
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
1040
+ stacked_params_mapping = [
1041
+ # (param_name, shard_name, shard_id)
1042
+ ("qkv_proj", "q_proj", "q"),
1043
+ ("qkv_proj", "k_proj", "k"),
1044
+ ("qkv_proj", "v_proj", "v"),
1045
+ ("gate_up_proj", "up_proj", 1),
1046
+ ("gate_up_proj", "gate_proj", 0),
1047
+ ]
1048
+ params_dict = dict(self.named_parameters(remove_duplicate=False))
1049
+ for name, loaded_weight in weights:
1050
+ if "rotary_emb.inv_freq" in name:
1051
+ continue
1052
+ if self.config.tie_word_embeddings and "lm_head.weight" in name:
1053
+ continue
1054
+ for (param_name, weight_name, shard_id) in stacked_params_mapping:
1055
+ if weight_name not in name:
1056
+ continue
1057
+ name = name.replace(weight_name, param_name)
1058
+ param = params_dict[name]
1059
+ weight_loader = param.weight_loader
1060
+ weight_loader(param, loaded_weight, shard_id)
1061
+ break
1062
+ else:
1063
+ if "visual" in name and "qkv.weight" in name:
1064
+ visual_num_heads = self.config.vision_config.num_heads
1065
+ visual_embed_dim = self.config.vision_config.embed_dim
1066
+ head_size = visual_embed_dim // visual_num_heads
1067
+ loaded_weight = loaded_weight.view(3, visual_num_heads,
1068
+ head_size,
1069
+ visual_embed_dim)
1070
+ loaded_weight = loaded_weight.transpose(0, 1)
1071
+ loaded_weight = loaded_weight.reshape(-1, visual_embed_dim)
1072
+ elif "visual" in name and "qkv.bias" in name:
1073
+ visual_num_heads = self.config.vision_config.num_heads
1074
+ visual_embed_dim = self.config.vision_config.embed_dim
1075
+ head_size = visual_embed_dim // visual_num_heads
1076
+ loaded_weight = loaded_weight.view(3, visual_num_heads,
1077
+ head_size)
1078
+ loaded_weight = loaded_weight.transpose(0, 1)
1079
+ loaded_weight = loaded_weight.reshape(-1)
1080
+ try:
1081
+ param = params_dict[name]
1082
+ except KeyError:
1083
+ print(params_dict.keys())
1084
+ raise
1085
+
1086
+ weight_loader = getattr(param, "weight_loader",
1087
+ default_weight_loader)
1088
+ weight_loader(param, loaded_weight)
vllm/multimodal/base.py
+3
-5
@@ -79,14 +79,12 @@ class MultiModalInputs(_MultiModalInputsBase):
79
if len(inputs_list) == 0:
80
return {}
81
82
- keys = inputs_list[0].keys()
83
-
82
item_lists: Dict[str, List[NestedTensors]] = defaultdict(list)
83
84
for inputs in inputs_list:
87
- if inputs.keys() != keys:
88
- msg = f"Inputs do not share the same keys ({keys})"
89
- raise ValueError(msg)
85
+ # For models that supports multiple modalities (e.g. Qwen2-VL),
86
+ # different modalities will return different data keys,
87
+ # so batch() should skip the same key check.
88
89
for k, v in inputs.items():
90
item_lists[k].append(v)
vllm/sequence.py
+11
@@ -165,6 +165,9 @@ class SequenceData(msgspec.Struct,
165
# is called.
166
_new_appended_tokens: List[int] = msgspec.field(default_factory=list)
167
168
+ # It is used to compute mrope_position_ids.
169
+ _mrope_position_delta: Optional[int] = None
170
+
171
def __post_init__(self) -> None:
172
assert self._prompt_token_ids.typecode == "l"
173
assert self._output_token_ids.typecode == "l"
@@ -219,6 +222,14 @@ class SequenceData(msgspec.Struct,
222
assert isinstance(self._output_token_ids, array)
223
return self._output_token_ids
224
225
+ @property
226
+ def mrope_position_delta(self) -> Optional[int]:
227
+ return self._mrope_position_delta
228
+
229
+ @mrope_position_delta.setter
230
+ def mrope_position_delta(self, new_mrope_position_delta):
231
+ self._mrope_position_delta = new_mrope_position_delta
232
+
233
def append_token_id(self, token_id: int, logprob: float) -> None:
234
self._output_token_ids.append(token_id)
235
self._new_appended_tokens.append(token_id)
vllm/transformers_utils/processor.py
new
+37
@@ -0,0 +1,37 @@
1
+from typing import cast
2
+
3
+
4
+def get_processor(
5
+ processor_name: str,
6
+ *args,
7
+ trust_remote_code: bool = False,
8
+ **kwargs,
9
+):
10
+ """Gets a processor for the given model name via HuggingFace."""
11
+ # don't put this import at the top level
12
+ # it will call torch.cuda.device_count()
13
+ from transformers import AutoProcessor
14
+ from transformers.processing_utils import ProcessorMixin
15
+
16
+ try:
17
+ processor = AutoProcessor.from_pretrained(
18
+ processor_name,
19
+ *args,
20
+ trust_remote_code=trust_remote_code,
21
+ **kwargs)
22
+ except ValueError as e:
23
+ # If the error pertains to the processor class not existing or not
24
+ # currently being imported, suggest using the --trust-remote-code flag.
25
+ # Unlike AutoTokenizer, AutoProcessor does not separate such errors
26
+ if not trust_remote_code:
27
+ err_msg = (
28
+ "Failed to load the processor. If the processor is "
29
+ "a custom processor not yet available in the HuggingFace "
30
+ "transformers library, consider setting "
31
+ "`trust_remote_code=True` in LLM or using the "
32
+ "`--trust-remote-code` flag in the CLI.")
33
+ raise RuntimeError(err_msg) from e
34
+ else:
35
+ raise e
36
+
37
+ return cast(ProcessorMixin, processor)
vllm/worker/model_runner.py
+99
-11
@@ -30,6 +30,7 @@ from vllm.lora.layers import LoRAMapping
30
from vllm.lora.request import LoRARequest
31
from vllm.lora.worker_manager import LRUCacheWorkerLoRAManager
32
from vllm.model_executor import SamplingMetadata, SamplingMetadataCache
33
+from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding
34
from vllm.model_executor.layers.sampler import SamplerOutput
35
from vllm.model_executor.model_loader import get_model
36
from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
@@ -181,6 +182,7 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
182
def simple_reinit(self):
183
self.input_tokens[0].clear() # type: ignore
184
self.input_positions[0].clear() # type: ignore
185
+ self.mrope_input_positions = None # type: ignore
186
self.seq_lens[0] = 0 # type: ignore
187
self.orig_seq_lens[0] = 0 # type: ignore
188
self.query_lens[0] = 0 # type: ignore
@@ -206,6 +208,7 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
208
# Input tokens and positions.
209
input_tokens: Optional[List[List[int]]] = None,
210
input_positions: Optional[List[List[int]]] = None,
211
+ mrope_input_positions: Optional[List[List[List[int]]]] = None,
212
213
# The sequence length (may be capped to the sliding window).
214
seq_lens: Optional[List[int]] = None,
@@ -266,6 +269,8 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
269
for seq_id in range(len(self.seq_ids)):
270
self.input_positions[seq_id].clear()
271
272
+ self.mrope_input_positions = None
273
+
274
if seq_lens:
275
self.seq_lens = seq_lens
276
else:
@@ -327,6 +332,7 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
332
else:
333
self.input_tokens = input_tokens or []
334
self.input_positions = input_positions or []
335
+ self.mrope_input_positions = mrope_input_positions or None
336
self.seq_lens = seq_lens or []
337
self.orig_seq_lens = orig_seq_lens or []
338
self.query_lens = query_lens or []
@@ -357,6 +363,7 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
363
364
self.input_tokens = [[] for _ in range(self.n_seqs)]
365
self.input_positions = [[] for _ in range(self.n_seqs)]
366
+ self.mrope_input_positions = None
367
self.seq_lens = [0] * self.n_seqs
368
self.orig_seq_lens = [0] * self.n_seqs
369
self.query_lens = [0] * self.n_seqs
@@ -493,6 +500,17 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
500
inter_data.query_lens[
501
seq_idx] = seq_len - context_len if inter_data.is_prompt else 1
502
503
+ if seq_data.mrope_position_delta is not None:
504
+ if inter_data.mrope_input_positions is None:
505
+ inter_data.mrope_input_positions = [None] * inter_data.n_seqs
506
+
507
+ inter_data.mrope_input_positions[
508
+ seq_idx] = MRotaryEmbedding.get_next_input_positions(
509
+ seq_data.mrope_position_delta,
510
+ context_len,
511
+ seq_len,
512
+ )
513
+
514
def _compute_for_prefix_cache_hit(
515
self, inter_data: InterDataForSeqGroup, seq_idx: int,
516
seq_group_metadata: SequenceGroupMetadata):
@@ -636,6 +654,40 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
654
mm_kwargs = self.multi_modal_input_mapper(mm_data)
655
inter_data.multi_modal_inputs = mm_kwargs
656
657
+ # special processing for mrope position deltas.
658
+ if self.runner.model_is_mrope:
659
+ image_grid_thw = mm_kwargs.get("image_grid_thw", None)
660
+ video_grid_thw = mm_kwargs.get("video_grid_thw", None)
661
+ assert image_grid_thw is not None or video_grid_thw is not None, (
662
+ "mrope embedding type requires multi-modal input mapper "
663
+ "returns 'image_grid_thw' or 'video_grid_thw'.")
664
+
665
+ hf_config = self.runner.model_config.hf_config
666
+
667
+ inter_data.mrope_input_positions = [None] * inter_data.n_seqs
668
+ for seq_idx in range(inter_data.n_seqs):
669
+ seq_data = seq_group_metadata.seq_data[
670
+ inter_data.seq_ids[seq_idx]]
671
+ token_ids = seq_data.get_token_ids()
672
+
673
+ mrope_input_positions, mrope_position_delta = \
674
+ MRotaryEmbedding.get_input_positions(
675
+ token_ids,
676
+ image_grid_thw=image_grid_thw,
677
+ video_grid_thw=video_grid_thw,
678
+ image_token_id=hf_config.image_token_id,
679
+ video_token_id=hf_config.video_token_id,
680
+ vision_start_token_id=hf_config.vision_start_token_id,
681
+ vision_end_token_id=hf_config.vision_end_token_id,
682
+ spatial_merge_size=hf_config.vision_config.
683
+ spatial_merge_size,
684
+ context_len=inter_data.context_lens[seq_idx],
685
+ )
686
+
687
+ seq_data.mrope_position_delta = mrope_position_delta
688
+ inter_data.mrope_input_positions[
689
+ seq_idx] = mrope_input_positions
690
+
691
def add_seq_group(self, seq_group_metadata: SequenceGroupMetadata):
692
"""Add a sequence group to the builder."""
693
seq_ids = seq_group_metadata.seq_data.keys()
@@ -684,10 +736,27 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
736
# prefix caching and there is no decode request.
737
return self.model_input_cls()
738
687
- input_positions = []
688
- for inter_data in self.inter_data_list:
689
- for cur_input_positions in inter_data.input_positions:
690
- input_positions.extend(cur_input_positions)
739
+ mrope_input_positions: Optional[List[List[int]]] = None
740
+ if any(inter_data.mrope_input_positions is not None
741
+ for inter_data in self.inter_data_list):
742
+ mrope_input_positions = [[] for _ in range(3)]
743
+ for idx in range(3):
744
+ for inter_data in self.inter_data_list:
745
+ msections = inter_data.mrope_input_positions
746
+ if msections is None:
747
+ for _seq_input_positions in inter_data.input_positions:
748
+ mrope_input_positions[idx].extend(
749
+ _seq_input_positions)
750
+ else:
751
+ for _seq_mrope_input_positions in msections:
752
+ mrope_input_positions[idx].extend(
753
+ _seq_mrope_input_positions[idx])
754
+ input_positions = None
755
+ else:
756
+ input_positions = []
757
+ for inter_data in self.inter_data_list:
758
+ for cur_input_positions in inter_data.input_positions:
759
+ input_positions.extend(cur_input_positions)
760
761
seq_lens = []
762
max_decode_seq_len = 0
@@ -724,15 +793,24 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
793
# Tokens and positions.
794
if cuda_graph_pad_size:
795
input_tokens.extend(itertools.repeat(0, cuda_graph_pad_size))
727
- input_positions.extend(itertools.repeat(0, cuda_graph_pad_size))
796
assert self.runner.device is not None
797
input_tokens_tensor = async_tensor_h2d(input_tokens, torch.long,
798
self.runner.device,
799
self.runner.pin_memory)
732
- input_positions_tensor = async_tensor_h2d(input_positions, torch.long,
733
- self.runner.device,
734
- self.runner.pin_memory)
735
-
800
+ if mrope_input_positions is not None:
801
+ for idx in range(3):
802
+ mrope_input_positions[idx].extend(
803
+ itertools.repeat(0, cuda_graph_pad_size))
804
+ input_positions_tensor = async_tensor_h2d(mrope_input_positions,
805
+ torch.long,
806
+ self.runner.device,
807
+ self.runner.pin_memory)
808
+ else:
809
+ input_positions.extend(itertools.repeat(0, cuda_graph_pad_size))
810
+ input_positions_tensor = async_tensor_h2d(input_positions,
811
+ torch.long,
812
+ self.runner.device,
813
+ self.runner.pin_memory)
814
# Sequence and query lengths.
815
if cuda_graph_pad_size:
816
seq_lens.extend(itertools.repeat(1, cuda_graph_pad_size))
@@ -1199,6 +1277,15 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1277
raise RuntimeError("PromptAdapter is not enabled.")
1278
return self.prompt_adapter_manager.list_adapters()
1279
1280
+ @property
1281
+ def model_is_mrope(self) -> bool:
1282
+ """Detect if the model has "mrope" rope_scaling type.
1283
+ mrope requires keep "rope_deltas" between prompt and decoding phases."""
1284
+ rope_scaling = getattr(self.model_config.hf_config, "rope_scaling", {})
1285
+ if rope_scaling is None:
1286
+ return False
1287
+ return rope_scaling.get("type", None) == "mrope"
1288
+
1289
@torch.inference_mode()
1290
def capture_model(self, kv_caches: List[List[torch.Tensor]]) -> None:
1291
"""Cuda graph capture a model.
@@ -1229,7 +1316,8 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1316
max_batch_size = self.max_batchsize_to_capture
1317
input_tokens = torch.zeros(max_batch_size, dtype=torch.long).cuda()
1318
input_positions = torch.zeros(max_batch_size, dtype=torch.long).cuda()
1232
-
1319
+ if self.model_is_mrope:
1320
+ input_positions = torch.tile(input_positions, (3, 1))
1321
# Prepare dummy previous_hidden_states only if needed by the model.
1322
# This is used by draft models such as EAGLE.
1323
previous_hidden_states = None
@@ -1293,7 +1381,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1381
"input_ids":
1382
input_tokens[:batch_size],
1383
"positions":
1296
- input_positions[:batch_size],
1384
+ input_positions[..., :batch_size],
1385
"hidden_or_intermediate_states":
1386
hidden_or_intermediate_states[
1387
virtual_engine] # type: ignore