@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 3b7fea770f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Model][VLM] Add Qwen2-VL model support (#7905)

Co-authored-by: Roger Wang <136131678+ywang96@users.noreply.github.com> Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>

Yang Fan committed Sep 12, 2024 at 00:31 UTC 3b7fea770f44369d077e40010bb4983ff3641535
14 files changed +1531 -31
docs/source/models/supported_models.rst
+7 -3
@@ -252,6 +252,11 @@ Multimodal Language Models
252 - Image\ :sup:`E`
253 - :code:`Qwen/Qwen-VL`, :code:`Qwen/Qwen-VL-Chat`, etc.
254 -
255 + * - :code:`Qwen2VLForConditionalGeneration`
256 + - Qwen2-VL (see note)
257 + - Image\ :sup:`+` / Video\ :sup:`+`
258 + - :code:`Qwen/Qwen2-VL-2B-Instruct`, :code:`Qwen/Qwen2-VL-7B-Instruct`, :code:`Qwen/Qwen2-VL-72B-Instruct`, etc.
259 + -
260 * - :code:`UltravoxModel`
261 - Ultravox
262 - Audio\ :sup:`E+`
@@ -265,15 +270,14 @@ Multimodal Language Models
270 For :code:`openbmb/MiniCPM-V-2`, the official repo doesn't work yet, so we need to use a fork (:code:`HwwwH/MiniCPM-V-2`) for now.
271 For more details, please see: https://github.com/vllm-project/vllm/pull/4087#issuecomment-2250397630
272
268 - For :code:`LLaVA-NeXT-Video`, the latest release of :code:`huggingface/transformers` doesn't work yet, so we need to use a developer version (:code:`21fac7abba2a37fae86106f87fcf9974fd1e3830`) for now.
273 +.. note::
274 + For :code:`LLaVA-NeXT-Video` and :code:`Qwen2-VL`, the latest release of :code:`huggingface/transformers` doesn't work yet, so we need to use a developer version (:code:`21fac7abba2a37fae86106f87fcf9974fd1e3830`) for now.
275 This can be installed by running the following command:
276
271 -
277 .. code-block:: bash
278
279 pip install git+https://github.com/huggingface/transformers.git@21fac7abba2a37fae86106f87fcf9974fd1e3830
280
276 -
281 ----
282
283 If your model uses one of the above model architectures, you can seamlessly run your model with vLLM.
examples/offline_inference_vision_language.py
+18
@@ -179,6 +179,23 @@ def run_qwen_vl(question):
179 return llm, prompt, stop_token_ids
180
181
182 +# Qwen2-VL
183 +def run_qwen2_vl(question):
184 + model_name = "Qwen/Qwen2-VL-7B-Instruct"
185 +
186 + llm = LLM(
187 + model=model_name,
188 + max_num_seqs=5,
189 + )
190 +
191 + prompt = ("<|im_start|>system\nYou are a helpful assistant.<|im_end|>\n"
192 + "<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>"
193 + f"{question}<|im_end|>\n"
194 + "<|im_start|>assistant\n")
195 + stop_token_ids = None
196 + return llm, prompt, stop_token_ids
197 +
198 +
199 model_example_map = {
200 "llava": run_llava,
201 "llava-next": run_llava_next,
@@ -191,6 +208,7 @@ model_example_map = {
208 "blip-2": run_blip2,
209 "internvl_chat": run_internvl,
210 "qwen_vl": run_qwen_vl,
211 + "qwen2_vl": run_qwen2_vl,
212 }
213
214
examples/offline_inference_vision_language_multi_image.py
+61 -7
@@ -6,7 +6,7 @@ by the model.
6 from argparse import Namespace
7 from typing import List
8
9 -from transformers import AutoTokenizer
9 +from transformers import AutoProcessor, AutoTokenizer
10
11 from vllm import LLM, SamplingParams
12 from vllm.multimodal.utils import fetch_image
@@ -30,7 +30,7 @@ def load_phi3v(question, image_urls: List[str]):
30 for i, _ in enumerate(image_urls, start=1))
31 prompt = f"<|user|>\n{placeholders}\n{question}<|end|>\n<|assistant|>\n"
32 stop_token_ids = None
33 - return llm, prompt, stop_token_ids
33 + return llm, prompt, stop_token_ids, None
34
35
36 def load_internvl(question, image_urls: List[str]):
@@ -60,18 +60,72 @@ def load_internvl(question, image_urls: List[str]):
60 # https://huggingface.co/OpenGVLab/InternVL2-2B#service
61 stop_tokens = ["<|endoftext|>", "<|im_start|>", "<|im_end|>", "<|end|>"]
62 stop_token_ids = [tokenizer.convert_tokens_to_ids(i) for i in stop_tokens]
63 - return llm, prompt, stop_token_ids
63 +
64 + return llm, prompt, stop_token_ids, None
65 +
66 +
67 +def load_qwen2_vl(question, image_urls: List[str]):
68 + try:
69 + from qwen_vl_utils import process_vision_info
70 + except ModuleNotFoundError:
71 + print('WARNING: `qwen-vl-utils` not installed, input images will not '
72 + 'be automatically resized. You can enable this functionality by '
73 + '`pip install qwen-vl-utils`.')
74 + process_vision_info = None
75 +
76 + model_name = "Qwen/Qwen2-VL-7B-Instruct"
77 +
78 + llm = LLM(
79 + model=model_name,
80 + max_num_seqs=5,
81 + max_model_len=32768 if process_vision_info is None else 4096,
82 + limit_mm_per_prompt={"image": len(image_urls)},
83 + )
84 +
85 + placeholders = [{"type": "image", "image": url} for url in image_urls]
86 + messages = [{
87 + "role": "system",
88 + "content": "You are a helpful assistant."
89 + }, {
90 + "role":
91 + "user",
92 + "content": [
93 + *placeholders,
94 + {
95 + "type": "text",
96 + "text": question
97 + },
98 + ],
99 + }]
100 +
101 + processor = AutoProcessor.from_pretrained(model_name)
102 +
103 + prompt = processor.apply_chat_template(messages,
104 + tokenize=False,
105 + add_generation_prompt=True)
106 +
107 + stop_token_ids = None
108 +
109 + if process_vision_info is None:
110 + image_data = [fetch_image(url) for url in image_urls]
111 + else:
112 + image_data, _ = process_vision_info(messages)
113 +
114 + return llm, prompt, stop_token_ids, image_data
115
116
117 model_example_map = {
118 "phi3_v": load_phi3v,
119 "internvl_chat": load_internvl,
120 + "qwen2_vl": load_qwen2_vl,
121 }
122
123
124 def run_generate(model, question: str, image_urls: List[str]):
73 - llm, prompt, stop_token_ids = model_example_map[model](question,
74 - image_urls)
125 + llm, prompt, stop_token_ids, image_data = model_example_map[model](
126 + question, image_urls)
127 + if image_data is None:
128 + image_data = [fetch_image(url) for url in image_urls]
129
130 sampling_params = SamplingParams(temperature=0.0,
131 max_tokens=128,
@@ -81,7 +135,7 @@ def run_generate(model, question: str, image_urls: List[str]):
135 {
136 "prompt": prompt,
137 "multi_modal_data": {
84 - "image": [fetch_image(url) for url in image_urls]
138 + "image": image_data
139 },
140 },
141 sampling_params=sampling_params)
@@ -92,7 +146,7 @@ def run_generate(model, question: str, image_urls: List[str]):
146
147
148 def run_chat(model: str, question: str, image_urls: List[str]):
95 - llm, _, stop_token_ids = model_example_map[model](question, image_urls)
149 + llm, _, stop_token_ids, _ = model_example_map[model](question, image_urls)
150
151 sampling_params = SamplingParams(temperature=0.0,
152 max_tokens=128,
requirements-common.txt
+1
@@ -28,3 +28,4 @@ importlib_metadata
28 mistral_common >= 1.3.4
29 pyyaml
30 six>=1.16.0; python_version > '3.11' # transitive dependency of pandas that needs to be the latest version for python 3.12
31 +einops # Required for Qwen2-VL.
tests/models/test_registry.py
+5
@@ -1,9 +1,14 @@
1 import pytest
2 +import transformers
3
4 from vllm.model_executor.models import _MODELS, ModelRegistry
5
6
7 @pytest.mark.parametrize("model_cls", _MODELS)
8 def test_registry_imports(model_cls):
9 + if (model_cls == "Qwen2VLForConditionalGeneration"
10 + and transformers.__version__ < "4.45"):
11 + pytest.skip("Waiting for next transformers release")
12 +
13 # Ensure all model classes can be imported successfully
14 ModelRegistry.resolve_model_cls([model_cls])
vllm/config.py
+6 -3
@@ -773,7 +773,7 @@ class LoadConfig:
773 ignore_patterns: The list of patterns to ignore when loading the model.
774 Default to "original/**/*" to avoid repeated loading of llama's
775 checkpoints.
776 -
776 +
777 """
778
779 load_format: Union[str, LoadFormat, "BaseModelLoader"] = LoadFormat.AUTO
@@ -1733,8 +1733,11 @@ def _get_and_verify_max_len(
1733 "with rope_scaling. Please raise an issue so we can "
1734 "investigate.")
1735
1736 - assert "factor" in rope_scaling
1737 - scaling_factor = rope_scaling["factor"]
1736 + if rope_type == "mrope":
1737 + scaling_factor = 1
1738 + else:
1739 + assert "factor" in rope_scaling
1740 + scaling_factor = rope_scaling["factor"]
1741 if rope_type == "yarn":
1742 derived_max_model_len = rope_scaling[
1743 "original_max_position_embeddings"]
vllm/entrypoints/chat_utils.py
+7 -1
@@ -108,7 +108,7 @@ class ConversationMessage(TypedDict, total=False):
108 """The tool calls generated by the model, such as function calls."""
109
110
111 -ModalityStr = Literal["image", "audio"]
111 +ModalityStr = Literal["image", "audio", "video"]
112 _T = TypeVar("_T")
113
114
@@ -158,12 +158,18 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
158 hf_config.image_token_index)
159 if model_type in ("chameleon", "internvl_chat"):
160 return "<image>"
161 + if model_type == "qwen2_vl":
162 + return "<|vision_start|><|image_pad|><|vision_end|>"
163
164 raise TypeError(f"Unknown model type: {model_type}")
165 elif modality == "audio":
166 if model_type == "ultravox":
167 return "<|reserved_special_token_0|>"
168 raise TypeError(f"Unknown model type: {model_type}")
169 + elif modality == "video":
170 + if model_type == "qwen2_vl":
171 + return "<|vision_start|><|video_pad|><|vision_end|>"
172 + raise TypeError(f"Unknown model type: {model_type}")
173 else:
174 raise TypeError(f"Unknown modality: {modality}")
175
vllm/model_executor/layers/rotary_embedding.py
+184 -1
@@ -712,6 +712,179 @@ class Llama3RotaryEmbedding(RotaryEmbedding):
712 return new_freqs
713
714
715 +class MRotaryEmbedding(RotaryEmbedding):
716 + """Rotary Embedding with Multimodal Sections."""
717 +
718 + def __init__(
719 + self,
720 + head_size: int,
721 + rotary_dim: int,
722 + max_position_embeddings: int,
723 + base: int,
724 + is_neox_style: bool,
725 + dtype: torch.dtype,
726 + mrope_section: Optional[List[int]] = None,
727 + ) -> None:
728 + super().__init__(head_size, rotary_dim, max_position_embeddings, base,
729 + is_neox_style, dtype)
730 +
731 + self.mrope_section = mrope_section
732 + if self.mrope_section:
733 + assert sum(self.mrope_section) == rotary_dim // 2
734 +
735 + def forward(
736 + self,
737 + positions: torch.Tensor,
738 + query: torch.Tensor,
739 + key: torch.Tensor,
740 + ) -> Tuple[torch.Tensor, torch.Tensor]:
741 + """PyTorch-native implementation equivalent to forward().
742 +
743 + Args:
744 + positions:
745 + [num_tokens,] (text only) or
746 + [3, num_tokens] (T/H/W positions with multimodal inputs)
747 + query: [num_tokens, num_heads * head_size]
748 + key: [num_tokens, num_kv_heads * head_size]
749 + """
750 + assert positions.ndim == 1 or positions.ndim == 2
751 +
752 + num_tokens = positions.shape[-1]
753 + cos_sin = self.cos_sin_cache[positions]
754 + cos, sin = cos_sin.chunk(2, dim=-1)
755 + if positions.ndim == 2:
756 + assert self.mrope_section
757 +
758 + cos = torch.cat([
759 + m[i]
760 + for i, m in enumerate(cos.split(self.mrope_section, dim=-1))
761 + ],
762 + dim=-1)
763 + sin = torch.cat([
764 + m[i]
765 + for i, m in enumerate(sin.split(self.mrope_section, dim=-1))
766 + ],
767 + dim=-1)
768 +
769 + query_shape = query.shape
770 + query = query.view(num_tokens, -1, self.head_size)
771 + query_rot = query[..., :self.rotary_dim]
772 + query_pass = query[..., self.rotary_dim:]
773 + query_rot = _apply_rotary_emb(query_rot, cos, sin, self.is_neox_style)
774 + query = torch.cat((query_rot, query_pass), dim=-1).reshape(query_shape)
775 +
776 + key_shape = key.shape
777 + key = key.view(num_tokens, -1, self.head_size)
778 + key_rot = key[..., :self.rotary_dim]
779 + key_pass = key[..., self.rotary_dim:]
780 + key_rot = _apply_rotary_emb(key_rot, cos, sin, self.is_neox_style)
781 + key = torch.cat((key_rot, key_pass), dim=-1).reshape(key_shape)
782 + return query, key
783 +
784 + @staticmethod
785 + def get_input_positions(
786 + input_tokens: List[int],
787 + image_grid_thw: Union[List[List[int]], torch.Tensor],
788 + video_grid_thw: Union[List[List[int]], torch.Tensor],
789 + image_token_id: int,
790 + video_token_id: int,
791 + vision_start_token_id: int,
792 + vision_end_token_id: int,
793 + spatial_merge_size: int,
794 + context_len: int = 0,
795 + ) -> Tuple[List[List[int]], int]:
796 + """Get mrope input positions and delta value."""
797 +
798 + if isinstance(image_grid_thw, torch.Tensor):
799 + image_grid_thw = image_grid_thw.tolist()
800 + if isinstance(video_grid_thw, torch.Tensor):
801 + video_grid_thw = video_grid_thw.tolist()
802 +
803 + input_tokens_tensor = torch.tensor(input_tokens)
804 + vision_start_indices = torch.argwhere(
805 + input_tokens_tensor == vision_start_token_id).squeeze(1)
806 + vision_tokens = input_tokens_tensor[vision_start_indices + 1]
807 + image_nums = (vision_tokens == image_token_id).sum()
808 + video_nums = (vision_tokens == video_token_id).sum()
809 + llm_pos_ids_list: list = []
810 +
811 + st = 0
812 + remain_images, remain_videos = image_nums, video_nums
813 +
814 + image_index, video_index = 0, 0
815 + for _ in range(image_nums + video_nums):
816 + if image_token_id in input_tokens and remain_images > 0:
817 + ed_image = input_tokens.index(image_token_id, st)
818 + else:
819 + ed_image = len(input_tokens) + 1
820 + if video_token_id in input_tokens and remain_videos > 0:
821 + ed_video = input_tokens.index(video_token_id, st)
822 + else:
823 + ed_video = len(input_tokens) + 1
824 + if ed_image < ed_video:
825 + t, h, w = (
826 + image_grid_thw[image_index][0],
827 + image_grid_thw[image_index][1],
828 + image_grid_thw[image_index][2],
829 + )
830 + image_index += 1
831 + remain_images -= 1
832 + ed = ed_image
833 + else:
834 + t, h, w = (
835 + video_grid_thw[video_index][0],
836 + video_grid_thw[video_index][1],
837 + video_grid_thw[video_index][2],
838 + )
839 + video_index += 1
840 + remain_videos -= 1
841 + ed = ed_video
842 + llm_grid_t, llm_grid_h, llm_grid_w = \
843 + t, h // spatial_merge_size, w // spatial_merge_size
844 + text_len = ed - st
845 +
846 + st_idx = llm_pos_ids_list[-1].max() + 1 if len(
847 + llm_pos_ids_list) > 0 else 0
848 + llm_pos_ids_list.append(
849 + torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx)
850 +
851 + t_index = torch.arange(llm_grid_t).view(-1, 1).expand(
852 + -1, llm_grid_h * llm_grid_w).flatten()
853 + h_index = torch.arange(llm_grid_h).view(1, -1, 1).expand(
854 + llm_grid_t, -1, llm_grid_w).flatten()
855 + w_index = torch.arange(llm_grid_w).view(1, 1, -1).expand(
856 + llm_grid_t, llm_grid_h, -1).flatten()
857 + llm_pos_ids_list.append(
858 + torch.stack([t_index, h_index, w_index]) + text_len + st_idx)
859 + st = ed + llm_grid_t * llm_grid_h * llm_grid_w
860 +
861 + if st < len(input_tokens):
862 + st_idx = llm_pos_ids_list[-1].max() + 1 if len(
863 + llm_pos_ids_list) > 0 else 0
864 + text_len = len(input_tokens) - st
865 + llm_pos_ids_list.append(
866 + torch.arange(text_len).view(1, -1).expand(3, -1) + st_idx)
867 +
868 + llm_positions = torch.cat(llm_pos_ids_list, dim=1).reshape(3, -1)
869 + llm_positions = llm_positions[:, context_len:]
870 + mrope_position_delta = (llm_positions.max() + 1 -
871 + len(input_tokens)).item()
872 +
873 + return llm_positions.tolist(), mrope_position_delta
874 +
875 + @staticmethod
876 + def get_next_input_positions(
877 + mrope_position_delta: int,
878 + context_len: int,
879 + seq_len: int,
880 + ) -> List[List[int]]:
881 + return [
882 + list(
883 + range(context_len + mrope_position_delta,
884 + seq_len + mrope_position_delta)) for _ in range(3)
885 + ]
886 +
887 +
888 _ROPE_DICT: Dict[Tuple, RotaryEmbedding] = {}
889
890
@@ -752,7 +925,7 @@ def get_rope(
925 # The correct one should be "longrope" but keep "su" here
926 # for backward compatible
927 if scaling_type not in {"su", "longrope"}:
755 - scaling_factor = rope_scaling["factor"]
928 + scaling_factor = rope_scaling.get("factor", 1.0)
929 if scaling_type == "llama3":
930 low_freq_factor = rope_scaling["low_freq_factor"]
931 high_freq_factor = rope_scaling["high_freq_factor"]
@@ -816,6 +989,16 @@ def get_rope(
989 head_size, rotary_dim, max_position, original_max_position,
990 base, is_neox_style, dtype, short_factor, long_factor,
991 **extra_kwargs)
992 + elif scaling_type == "mrope":
993 + return MRotaryEmbedding(
994 + head_size,
995 + rotary_dim,
996 + max_position,
997 + base,
998 + is_neox_style,
999 + dtype,
1000 + mrope_section=rope_scaling["mrope_section"],
1001 + )
1002 else:
1003 raise ValueError(f"Unknown RoPE scaling type {scaling_type}")
1004 _ROPE_DICT[key] = rotary_emb
vllm/model_executor/models/__init__.py
+4
@@ -53,6 +53,8 @@ _GENERATION_MODELS = {
53 "PhiMoEForCausalLM": ("phimoe", "PhiMoEForCausalLM"),
54 "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
55 "Qwen2MoeForCausalLM": ("qwen2_moe", "Qwen2MoeForCausalLM"),
56 + "Qwen2VLForConditionalGeneration":
57 + ("qwen2_vl", "Qwen2VLForConditionalGeneration"),
58 "RWForCausalLM": ("falcon", "FalconForCausalLM"),
59 "StableLMEpochForCausalLM": ("stablelm", "StablelmForCausalLM"),
60 "StableLmForCausalLM": ("stablelm", "StablelmForCausalLM"),
@@ -90,6 +92,8 @@ _MULTIMODAL_MODELS = {
92 "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
93 "UltravoxModel": ("ultravox", "UltravoxModel"),
94 "QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
95 + "Qwen2VLForConditionalGeneration": ("qwen2_vl",
96 + "Qwen2VLForConditionalGeneration"),
97 }
98 _CONDITIONAL_GENERATION_MODELS = {
99 "BartModel": ("bart", "BartForConditionalGeneration"),
vllm/model_executor/models/qwen2_vl.py new
+1088
@@ -0,0 +1,1088 @@
1 +# coding=utf-8
2 +# Adapted from
3 +# https://github.com/huggingface/transformers/blob/19e6e80e10118f855137b90740936c0b11ac397f/src/transformers/models/qwen2_vl/modeling_qwen2_vl.py
4 +# Copyright 2024 The Qwen team.
5 +# Copyright 2023 The vLLM team.
6 +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
7 +#
8 +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
9 +# and OPT implementations in this library. It has been modified from its
10 +# original forms to accommodate minor architectural differences compared
11 +# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
12 +#
13 +# Licensed under the Apache License, Version 2.0 (the "License");
14 +# you may not use this file except in compliance with the License.
15 +# You may obtain a copy of the License at
16 +#
17 +# http://www.apache.org/licenses/LICENSE-2.0
18 +#
19 +# Unless required by applicable law or agreed to in writing, software
20 +# distributed under the License is distributed on an "AS IS" BASIS,
21 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
22 +# See the License for the specific language governing permissions and
23 +# limitations under the License.
24 +"""Inference-only Qwen2-VL model compatible with HuggingFace weights."""
25 +from array import array
26 +from functools import lru_cache, partial
27 +from typing import (Iterable, List, Mapping, Optional, Tuple, Type, TypedDict,
28 + Union)
29 +
30 +import torch
31 +import torch.nn as nn
32 +import torch.nn.functional as F
33 +from einops import rearrange, repeat
34 +from PIL import Image
35 +from transformers import Qwen2VLConfig
36 +from transformers.image_utils import (get_image_size,
37 + infer_channel_dimension_format,
38 + to_numpy_array)
39 +from transformers.models.qwen2_vl.configuration_qwen2_vl import (
40 + Qwen2VLVisionConfig)
41 +from transformers.models.qwen2_vl.image_processing_qwen2_vl import (
42 + make_batched_images, make_batched_videos, smart_resize)
43 +
44 +import vllm.envs as envs
45 +from vllm.attention import AttentionMetadata
46 +from vllm.attention.selector import (_Backend, backend_name_to_enum,
47 + get_global_forced_attn_backend)
48 +from vllm.config import CacheConfig, MultiModalConfig
49 +from vllm.distributed import parallel_state
50 +from vllm.distributed import utils as dist_utils
51 +from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
52 +from vllm.logger import init_logger
53 +from vllm.model_executor import SamplingMetadata
54 +from vllm.model_executor.layers.activation import QuickGELU
55 +from vllm.model_executor.layers.linear import (ColumnParallelLinear,
56 + RowParallelLinear)
57 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
58 +from vllm.model_executor.layers.quantization import QuantizationConfig
59 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
60 +from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
61 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
62 +from vllm.model_executor.models.interfaces import SupportsMultiModal
63 +from vllm.model_executor.models.qwen2 import Qwen2Model
64 +from vllm.multimodal import (MULTIMODAL_REGISTRY, MultiModalDataDict,
65 + MultiModalInputs)
66 +from vllm.multimodal.base import MultiModalData
67 +from vllm.multimodal.image import cached_get_image_processor
68 +from vllm.platforms import current_platform
69 +from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
70 + SequenceData)
71 +from vllm.transformers_utils.processor import get_processor
72 +
73 +logger = init_logger(__name__)
74 +
75 +# === Vision Inputs === #
76 +
77 +
78 +class Qwen2VLImageInputs(TypedDict):
79 + pixel_values: torch.Tensor
80 + """Shape:
81 + `(num_patches, num_channels * patch_size * patch_size)`
82 + """
83 +
84 + image_grid_thw: torch.Tensor
85 + """Shape: `(num_images, 3)`
86 +
87 + This should be in `(grid_t, grid_h, grid_w)` format.
88 + """
89 +
90 +
91 +class Qwen2VLVideoInputs(TypedDict):
92 + pixel_values_videos: torch.Tensor
93 + """Shape:
94 + `(num_patches,
95 + num_channels * temporal_patch_size * patch_size * patch_size)`
96 + """
97 +
98 + video_grid_thw: torch.Tensor
99 + """Shape: `(num_videos, 3)`
100 +
101 + This should be in `(grid_t, grid_h, grid_w)` format.
102 + """
103 +
104 +
105 +# === Vision Encoder === #
106 +
107 +
108 +class Qwen2VisionMLP(nn.Module):
109 +
110 + def __init__(
111 + self,
112 + in_features: int,
113 + hidden_features: int = None,
114 + act_layer: Type[nn.Module] = QuickGELU,
115 + quant_config: Optional[QuantizationConfig] = None,
116 + ):
117 + super().__init__()
118 + self.fc1 = ColumnParallelLinear(in_features,
119 + hidden_features,
120 + quant_config=quant_config)
121 + self.act = act_layer()
122 + self.fc2 = RowParallelLinear(hidden_features,
123 + in_features,
124 + quant_config=quant_config)
125 +
126 + def forward(self, x: torch.Tensor) -> torch.Tensor:
127 + x_parallel, _ = self.fc1(x)
128 + x_parallel = self.act(x_parallel)
129 + x, _ = self.fc2(x_parallel)
130 + return x
131 +
132 +
133 +def rotate_half(x: torch.Tensor, interleaved: bool = False) -> torch.Tensor:
134 + if not interleaved:
135 + x1, x2 = x.chunk(2, dim=-1)
136 + return torch.cat((-x2, x1), dim=-1)
137 + else:
138 + x1, x2 = x[..., ::2], x[..., 1::2]
139 + return rearrange(torch.stack((-x2, x1), dim=-1),
140 + "... d two -> ... (d two)",
141 + two=2)
142 +
143 +
144 +def apply_rotary_emb_torch(x: torch.Tensor,
145 + cos: torch.Tensor,
146 + sin: torch.Tensor,
147 + interleaved: bool = False) -> torch.Tensor:
148 + """
149 + x: (batch_size, seqlen, nheads, headdim)
150 + cos, sin: (seqlen, rotary_dim / 2) or (batch_size, seqlen, rotary_dim / 2)
151 + """
152 + ro_dim = cos.shape[-1] * 2
153 + assert ro_dim <= x.shape[-1]
154 + cos = repeat(
155 + cos,
156 + "... d -> ... 1 (2 d)" if not interleaved else "... d -> ... 1 (d 2)")
157 + sin = repeat(
158 + sin,
159 + "... d -> ... 1 (2 d)" if not interleaved else "... d -> ... 1 (d 2)")
160 + return torch.cat(
161 + [
162 + x[..., :ro_dim] * cos +
163 + rotate_half(x[..., :ro_dim], interleaved) * sin, x[..., ro_dim:]
164 + ],
165 + dim=-1,
166 + )
167 +
168 +
169 +def apply_rotary_pos_emb_vision(t: torch.Tensor,
170 + freqs: torch.Tensor) -> torch.Tensor:
171 + t_ = t.float()
172 + cos = freqs.cos()
173 + sin = freqs.sin()
174 + output = apply_rotary_emb_torch(t_, cos, sin).type_as(t)
175 + return output
176 +
177 +
178 +class Qwen2VisionAttention(nn.Module):
179 +
180 + def __init__(
181 + self,
182 + embed_dim: Optional[int] = None,
183 + num_heads: Optional[int] = None,
184 + projection_size: Optional[int] = None,
185 + quant_config: Optional[QuantizationConfig] = None,
186 + ) -> None:
187 + super().__init__()
188 + # Per attention head and per partition values.
189 + world_size = parallel_state.get_tensor_model_parallel_world_size()
190 + self.hidden_size_per_attention_head = dist_utils.divide(
191 + projection_size, num_heads)
192 + self.num_attention_heads_per_partition = dist_utils.divide(
193 + num_heads, world_size)
194 +
195 + self.qkv = ColumnParallelLinear(input_size=embed_dim,
196 + output_size=3 * projection_size,
197 + quant_config=quant_config)
198 + self.proj = RowParallelLinear(input_size=projection_size,
199 + output_size=embed_dim,
200 + quant_config=quant_config)
201 +
202 + # Detect attention implementation.
203 + selected_backend: Optional[_Backend] = get_global_forced_attn_backend()
204 + if selected_backend is None:
205 + backend_by_env_var: Optional[str] = envs.VLLM_ATTENTION_BACKEND
206 + if backend_by_env_var is not None:
207 + selected_backend = backend_name_to_enum(backend_by_env_var)
208 + if selected_backend is None:
209 + # For Volta and Turing GPUs, use xformers instead.
210 + device_available = current_platform.get_device_capability()[0] >= 8
211 + if device_available:
212 + from transformers.utils import is_flash_attn_2_available
213 +
214 + if is_flash_attn_2_available():
215 + self._use_flash_attn = True
216 + else:
217 + logger.warning(
218 + "Current Qwen2-VL implementation has a bug with "
219 + "`vllm-flash-attn` inside vision module, so we use "
220 + "xformers backend instead. You can run `pip install "
221 + "flash-attn to use flash-attention backend.")
222 + self._use_flash_attn = False
223 + else:
224 + self._use_flash_attn = False
225 + else:
226 + if selected_backend == _Backend.FLASH_ATTN:
227 + self._use_flash_attn = True
228 + elif selected_backend == _Backend.XFORMERS:
229 + self._use_flash_attn = False
230 + else:
231 + raise RuntimeError(
232 + f"Qwen2-VL does not support {selected_backend} backend now."
233 + )
234 +
235 + def forward(
236 + self,
237 + x: torch.Tensor,
238 + cu_seqlens: torch.Tensor,
239 + rotary_pos_emb: torch.Tensor = None,
240 + ) -> torch.Tensor:
241 + # [s, b, c] --> [s, b, head * 3 * head_dim]
242 + x, _ = self.qkv(x)
243 +
244 + # [s, b, head * 3 * head_dim] --> [s, b, head, 3 * head_dim]
245 + new_x_shape = x.size()[:-1] + (
246 + self.num_attention_heads_per_partition,
247 + 3 * self.hidden_size_per_attention_head,
248 + )
249 + x = x.view(*new_x_shape)
250 +
251 + # [s, b, head, 3 * head_dim] --> 3 [s, b, head, head_dim]
252 + q, k, v = dist_utils.split_tensor_along_last_dim(x, 3)
253 + batch_size = q.shape[1]
254 +
255 + q, k, v = [
256 + rearrange(x, "s b ... -> b s ...").contiguous() for x in (q, k, v)
257 + ]
258 + if rotary_pos_emb is not None:
259 + q = apply_rotary_pos_emb_vision(q, rotary_pos_emb)
260 + k = apply_rotary_pos_emb_vision(k, rotary_pos_emb)
261 +
262 + if self._use_flash_attn:
263 + # from vllm_flash_attn.flash_attn_interface import (
264 + # flash_attn_varlen_func)
265 + from flash_attn import flash_attn_varlen_func
266 +
267 + q, k, v = [rearrange(x, "b s ... -> (b s) ...") for x in [q, k, v]]
268 +
269 + max_seqlen = (cu_seqlens[1:] - cu_seqlens[:-1]).max().item()
270 + output = flash_attn_varlen_func(q,
271 + k,
272 + v,
273 + cu_seqlens_q=cu_seqlens,
274 + cu_seqlens_k=cu_seqlens,
275 + max_seqlen_q=max_seqlen,
276 + max_seqlen_k=max_seqlen,
277 + dropout_p=0,
278 + causal=False)
279 +
280 + context_layer = rearrange(output,
281 + "(b s) ... -> b s ...",
282 + b=batch_size)
283 + else:
284 + from xformers import ops as xops
285 + from xformers.ops.fmha.attn_bias import BlockDiagonalMask
286 +
287 + seqlens = (cu_seqlens[1:] - cu_seqlens[:-1]).tolist()
288 + attn_bias = BlockDiagonalMask.from_seqlens(q_seqlen=seqlens,
289 + kv_seqlen=None)
290 +
291 + context_layer = xops.memory_efficient_attention_forward(
292 + q, k, v, attn_bias=attn_bias, p=0, scale=None)
293 + context_layer = rearrange(context_layer,
294 + "b s h d -> s b (h d)").contiguous()
295 +
296 + output, _ = self.proj(context_layer)
297 + return output
298 +
299 +
300 +class Qwen2VisionBlock(nn.Module):
301 +
302 + def __init__(
303 + self,
304 + dim: int,
305 + num_heads: int,
306 + mlp_ratio: float,
307 + act_layer: Type[nn.Module] = QuickGELU,
308 + norm_layer: Type[nn.Module] = None,
309 + quant_config: Optional[QuantizationConfig] = None,
310 + ) -> None:
311 + super().__init__()
312 + if norm_layer is None:
313 + norm_layer = partial(nn.LayerNorm, eps=1e-6)
314 + self.norm1 = norm_layer(dim)
315 + self.norm2 = norm_layer(dim)
316 + mlp_hidden_dim = int(dim * mlp_ratio)
317 +
318 + self.attn = Qwen2VisionAttention(embed_dim=dim,
319 + num_heads=num_heads,
320 + projection_size=dim,
321 + quant_config=quant_config)
322 + self.mlp = Qwen2VisionMLP(dim,
323 + mlp_hidden_dim,
324 + act_layer=act_layer,
325 + quant_config=quant_config)
326 +
327 + def forward(self, x: torch.Tensor, cu_seqlens: torch.Tensor,
328 + rotary_pos_emb: torch.Tensor) -> torch.Tensor:
329 + x = x + self.attn(self.norm1(x),
330 + cu_seqlens=cu_seqlens,
331 + rotary_pos_emb=rotary_pos_emb)
332 + x = x + self.mlp(self.norm2(x))
333 + return x
334 +
335 +
336 +class Qwen2VisionPatchEmbed(nn.Module):
337 +
338 + def __init__(
339 + self,
340 + patch_size: int = 14,
341 + temporal_patch_size: int = 2,
342 + in_chans: int = 3,
343 + embed_dim: int = 1152,
344 + ) -> None:
345 + super().__init__()
346 + self.patch_size = patch_size
347 + self.temporal_patch_size = temporal_patch_size
348 + self.embed_dim = embed_dim
349 +
350 + kernel_size = [temporal_patch_size, patch_size, patch_size]
351 + self.proj = nn.Conv3d(in_chans,
352 + embed_dim,
353 + kernel_size=kernel_size,
354 + stride=kernel_size,
355 + bias=False)
356 +
357 + def forward(self, x: torch.Tensor) -> torch.Tensor:
358 + L, C = x.shape
359 + x = x.view(L, -1, self.temporal_patch_size, self.patch_size,
360 + self.patch_size)
361 + x = self.proj(x).view(L, self.embed_dim)
362 + return x
363 +
364 +
365 +class Qwen2VisionPatchMerger(nn.Module):
366 +
367 + def __init__(
368 + self,
369 + d_model: int,
370 + context_dim: int,
371 + norm_layer: Type[nn.Module] = None,
372 + spatial_merge_size: int = 2,
373 + quant_config: Optional[QuantizationConfig] = None,
374 + ) -> None:
375 + super().__init__()
376 + self.hidden_size = context_dim * (spatial_merge_size**2)
377 + if norm_layer is None:
378 + norm_layer = partial(nn.LayerNorm, eps=1e-6)
379 + self.ln_q = norm_layer(context_dim)
380 + self.mlp = nn.ModuleList([
381 + ColumnParallelLinear(self.hidden_size,
382 + self.hidden_size,
383 + bias=True,
384 + quant_config=quant_config),
385 + nn.GELU(),
386 + RowParallelLinear(self.hidden_size,
387 + d_model,
388 + bias=True,
389 + quant_config=quant_config),
390 + ])
391 +
392 + def forward(self, x: torch.Tensor) -> torch.Tensor:
393 + x = self.ln_q(x)
394 + x = x.view(-1, self.hidden_size)
395 +
396 + mlp_fc1, mlp_act, mlp_fc2 = self.mlp
397 + x_parallel, _ = mlp_fc1(x)
398 + x_parallel = mlp_act(x_parallel)
399 + out, _ = mlp_fc2(x_parallel)
400 + return out
401 +
402 +
403 +class Qwen2VisionRotaryEmbedding(nn.Module):
404 +
405 + def __init__(self, dim: int, theta: float = 10000.0) -> None:
406 + super().__init__()
407 + self.dim = dim
408 + self.theta = theta
409 + inv_freq = 1.0 / (theta
410 + **(torch.arange(0, dim, 2, dtype=torch.float) / dim))
411 + self.register_buffer("inv_freq", inv_freq, persistent=False)
412 + self._seq_len_cached = 0
413 + self._freqs_cached = None
414 +
415 + def update_freqs_cache(self, seqlen: int) -> None:
416 + if seqlen > self._seq_len_cached:
417 + seqlen *= 2
418 + self._seq_len_cached = seqlen
419 + self.inv_freq = 1.0 / (self.theta**(torch.arange(
420 + 0, self.dim, 2, dtype=torch.float, device=self.inv_freq.device)
421 + / self.dim))
422 + seq = torch.arange(seqlen,
423 + device=self.inv_freq.device,
424 + dtype=self.inv_freq.dtype)
425 + freqs = torch.outer(seq, self.inv_freq)
426 + self._freqs_cached = freqs
427 +
428 + def forward(self, seqlen: int) -> torch.Tensor:
429 + self.update_freqs_cache(seqlen)
430 + return self._freqs_cached[:seqlen]
431 +
432 +
433 +class Qwen2VisionTransformer(nn.Module):
434 +
435 + def __init__(
436 + self,
437 + vision_config: Qwen2VLVisionConfig,
438 + norm_eps: float = 1e-6,
439 + quant_config: Optional[QuantizationConfig] = None,
440 + ) -> None:
441 + super().__init__()
442 +
443 + patch_size: int = vision_config.patch_size
444 + temporal_patch_size: int = vision_config.temporal_patch_size
445 + spatial_merge_size: int = vision_config.spatial_merge_size
446 + in_chans: int = vision_config.in_chans
447 + hidden_size: int = vision_config.hidden_size
448 + embed_dim: int = vision_config.embed_dim
449 + depth: int = vision_config.depth
450 + num_heads: int = vision_config.num_heads
451 + mlp_ratio: float = vision_config.mlp_ratio
452 +
453 + self.spatial_merge_size = spatial_merge_size
454 +
455 + self.patch_embed = Qwen2VisionPatchEmbed(
456 + patch_size=patch_size,
457 + temporal_patch_size=temporal_patch_size,
458 + in_chans=in_chans,
459 + embed_dim=embed_dim,
460 + )
461 +
462 + norm_layer = partial(nn.LayerNorm, eps=norm_eps)
463 + head_dim = embed_dim // num_heads
464 + self.rotary_pos_emb = Qwen2VisionRotaryEmbedding(head_dim // 2)
465 +
466 + self.blocks = nn.ModuleList([
467 + Qwen2VisionBlock(
468 + dim=embed_dim,
469 + num_heads=num_heads,
470 + mlp_ratio=mlp_ratio,
471 + norm_layer=norm_layer,
472 + quant_config=quant_config,
473 + ) for _ in range(depth)
474 + ])
475 + self.merger = Qwen2VisionPatchMerger(
476 + d_model=hidden_size,
477 + context_dim=embed_dim,
478 + norm_layer=norm_layer,
479 + quant_config=quant_config,
480 + )
481 +
482 + @property
483 + def dtype(self) -> torch.dtype:
484 + return self.blocks[0].mlp.fc2.weight.dtype
485 +
486 + @property
487 + def device(self) -> torch.device:
488 + return self.blocks[0].mlp.fc2.weight.device
489 +
490 + def rot_pos_emb(self, grid_thw: torch.Tensor) -> torch.Tensor:
491 + pos_ids = []
492 + for t, h, w in grid_thw:
493 + hpos_ids = torch.arange(h).unsqueeze(1).expand(-1, w)
494 + wpos_ids = torch.arange(w).unsqueeze(0).expand(h, -1)
495 + hpos_ids = hpos_ids.reshape(
496 + h // self.spatial_merge_size,
497 + self.spatial_merge_size,
498 + w // self.spatial_merge_size,
499 + self.spatial_merge_size,
500 + ).permute(0, 2, 1, 3).flatten()
501 + wpos_ids = wpos_ids.reshape(
502 + h // self.spatial_merge_size,
503 + self.spatial_merge_size,
504 + w // self.spatial_merge_size,
505 + self.spatial_merge_size,
506 + ).permute(0, 2, 1, 3).flatten()
507 + pos_ids.append(
508 + torch.stack([hpos_ids, wpos_ids], dim=-1).repeat(t, 1))
509 + pos_ids = torch.cat(pos_ids, dim=0)
510 + max_grid_size = grid_thw[:, 1:].max()
511 + rotary_pos_emb_full = self.rotary_pos_emb(max_grid_size)
512 + rotary_pos_emb = rotary_pos_emb_full[pos_ids].flatten(1)
513 + return rotary_pos_emb
514 +
515 + def forward(
516 + self,
517 + x: torch.Tensor,
518 + grid_thw: torch.Tensor,
519 + ) -> torch.Tensor:
520 + # patchify
521 + x = x.to(device=self.device, dtype=self.dtype)
522 + x = self.patch_embed(x)
523 +
524 + # compute position embedding
525 + rotary_pos_emb = self.rot_pos_emb(grid_thw)
526 +
527 + # compute cu_seqlens
528 + cu_seqlens = torch.repeat_interleave(grid_thw[:, 1] * grid_thw[:, 2],
529 + grid_thw[:, 0]).cumsum(
530 + dim=0, dtype=torch.int32)
531 + cu_seqlens = F.pad(cu_seqlens, (1, 0), "constant", 0)
532 +
533 + # transformers
534 + x = x.unsqueeze(1)
535 + for blk in self.blocks:
536 + x = blk(x, cu_seqlens=cu_seqlens, rotary_pos_emb=rotary_pos_emb)
537 +
538 + # adapter
539 + x = self.merger(x)
540 + return x
541 +
542 +
543 +# === Vision input helpers === #
544 +
545 +cached_get_processor = lru_cache(get_processor)
546 +
547 +
548 +def mm_input_mapper_for_qwen2_vl(
549 + ctx: InputContext,
550 + data: MultiModalData[object],
551 + data_type_key: str,
552 +) -> MultiModalInputs:
553 + """Input mapper for Qwen2-VL."""
554 + model_config = ctx.model_config
555 + image_processor = cached_get_image_processor(
556 + model_config.model, trust_remote_code=model_config.trust_remote_code)
557 + if image_processor is None:
558 + raise RuntimeError("No HuggingFace processor is available "
559 + "to process the image object")
560 +
561 + images = None
562 + videos = None
563 + if data_type_key == "image":
564 + images = data
565 + else:
566 + assert data_type_key == "video"
567 + videos = data
568 +
569 + try:
570 + batch_data = image_processor \
571 + .preprocess(images=images, videos=videos, return_tensors="pt") \
572 + .data
573 + except Exception:
574 + logger.error("Failed to process image (%s)", data)
575 + raise
576 +
577 + return MultiModalInputs(batch_data)
578 +
579 +
580 +image_input_mapper_for_qwen2_vl = partial(mm_input_mapper_for_qwen2_vl,
581 + data_type_key="image")
582 +video_input_mapper_for_qwen2_vl = partial(mm_input_mapper_for_qwen2_vl,
583 + data_type_key="video")
584 +
585 +
586 +def _get_vision_info(
587 + image_processor,
588 + height: int,
589 + width: int,
590 + min_pixels: int,
591 + max_pixels: int,
592 + do_resize: bool = True,
593 + data_type_key: str = "image",
594 + mm_count: int = 1,
595 +):
596 + """Get information (resized height / width and number of vision tokens)
597 + of input image / video frame."""
598 +
599 + if do_resize:
600 + resized_height, resized_width = smart_resize(
601 + height=height,
602 + width=width,
603 + factor=image_processor.patch_size * image_processor.merge_size,
604 + min_pixels=min_pixels,
605 + max_pixels=max_pixels,
606 + )
607 + else:
608 + resized_height, resized_width = height, width
609 +
610 + if data_type_key == "image":
611 + grid_t = mm_count
612 + else:
613 + assert data_type_key == "video"
614 + grid_t = max(mm_count // image_processor.temporal_patch_size, 1)
615 +
616 + grid_h = resized_height // image_processor.patch_size
617 + grid_w = resized_width // image_processor.patch_size
618 + vision_tokens = grid_t * grid_h * grid_w
619 + llm_num_vision_tokens = (vision_tokens // image_processor.merge_size //
620 + image_processor.merge_size)
621 +
622 + return resized_height, resized_width, llm_num_vision_tokens
623 +
624 +
625 +def _get_max_image_info(
626 + image_processor,
627 + data_type_key: str = "image",
628 + mm_count: int = 1,
629 +):
630 + return _get_vision_info(
631 + image_processor,
632 + height=9999999,
633 + width=9999999,
634 +
635 + # Limit min / max pixels.
636 + min_pixels=max(image_processor.min_pixels, 28 * 28),
637 + max_pixels=min(image_processor.max_pixels, 1280 * 28 * 28),
638 + data_type_key=data_type_key,
639 + mm_count=mm_count,
640 + )
641 +
642 +
643 +def get_max_qwen2_vl_mm_tokens(ctx: InputContext, data_type_key: str) -> int:
644 + image_processor = cached_get_image_processor(ctx.model_config.model)
645 + max_resized_height, max_resized_width, max_llm_image_tokens = \
646 + _get_max_image_info(image_processor, data_type_key=data_type_key,
647 + mm_count=1)
648 + return max_llm_image_tokens
649 +
650 +
651 +get_max_qwen2_vl_image_tokens = partial(get_max_qwen2_vl_mm_tokens,
652 + data_type_key="image")
653 +get_max_qwen2_vl_video_tokens = partial(get_max_qwen2_vl_mm_tokens,
654 + data_type_key="video")
655 +
656 +
657 +def dummy_data_for_qwen2_vl(
658 + ctx: InputContext, seq_len: int, mm_counts: Mapping[str, int]
659 +) -> Tuple[SequenceData, Optional[MultiModalDataDict]]:
660 + image_processor = cached_get_image_processor(ctx.model_config.model)
661 +
662 + num_images = mm_counts["image"]
663 + max_resized_height, max_resized_width, max_llm_image_tokens = \
664 + _get_max_image_info(image_processor, data_type_key="image",
665 + mm_count=num_images)
666 + if seq_len - max_llm_image_tokens - 2 < 0:
667 + raise RuntimeError(
668 + f"Qwen2-VL cannot process {num_images} images in a prompt, "
669 + "please increase max_model_len or reduce image limit by "
670 + "--limit-mm-per-prompt.")
671 +
672 + # Check video counts.
673 + num_videos = mm_counts["video"]
674 + max_resized_height, max_resized_width, max_llm_video_tokens = \
675 + _get_max_image_info(image_processor, data_type_key="video",
676 + mm_count=num_videos)
677 + if seq_len - max_llm_video_tokens - 2 < 0:
678 + raise RuntimeError(
679 + f"Qwen2-VL cannot process {num_images} videos in a prompt, "
680 + "please increase max_model_len or reduce video limit by "
681 + "--limit-mm-per-prompt.")
682 +
683 + hf_config = ctx.get_hf_config(Qwen2VLConfig)
684 + token_ids = array(VLLM_TOKEN_ID_ARRAY_TYPE,
685 + [hf_config.vision_start_token_id])
686 + token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE,
687 + [hf_config.image_token_id]) * max_llm_image_tokens
688 + token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE,
689 + [hf_config.vision_end_token_id])
690 + token_ids += array(VLLM_TOKEN_ID_ARRAY_TYPE,
691 + [0]) * (seq_len - max_llm_image_tokens - 2)
692 + dummy_seqdata = SequenceData(token_ids)
693 + dummy_image = Image.new("RGB", (max_resized_width, max_resized_height),
694 + color=0)
695 +
696 + return dummy_seqdata, {
697 + "image": dummy_image if num_images == 1 else [dummy_image] * num_images
698 + }
699 +
700 +
701 +def _get_llm_num_vision_tokens(
702 + mm_inputs: list,
703 + data_type_key: str,
704 + image_processor,
705 +):
706 + """Get number of vision tokens of multimodal inputs.
707 +
708 + This method is derived from `transformers.models.qwen2_vl.
709 + image_processing_qwen2_vl.Qwen2VLImageProcessor._preprocess`.
710 + """
711 + image = to_numpy_array(mm_inputs[0])
712 + input_data_format = infer_channel_dimension_format(image)
713 + height, width = get_image_size(image, channel_dim=input_data_format)
714 + _, _, llm_num_vision_tokens = _get_vision_info(
715 + image_processor,
716 + height=height,
717 + width=width,
718 + min_pixels=image_processor.min_pixels,
719 + max_pixels=image_processor.max_pixels,
720 + do_resize=image_processor.do_resize,
721 + data_type_key=data_type_key,
722 + mm_count=len(mm_inputs),
723 + )
724 + return llm_num_vision_tokens
725 +
726 +
727 +def input_processor_for_qwen2_vl(ctx: InputContext,
728 + llm_inputs: LLMInputs) -> LLMInputs:
729 + multi_modal_data = llm_inputs.get("multi_modal_data", None)
730 + if multi_modal_data is None:
731 + return llm_inputs
732 +
733 + image_inputs = multi_modal_data.get("image", None)
734 + video_inputs = multi_modal_data.get("video", None)
735 +
736 + processor = cached_get_processor(ctx.model_config.model)
737 + image_processor = processor.image_processor
738 + hf_config = ctx.get_hf_config(Qwen2VLConfig)
739 +
740 + # To avoid redundant processing of vision objects (resize, rescale, etc.),
741 + # we extract code of calculating number of vision tokens from
742 + # `transformers.models.qwen2_vl.processing_qwen2_vl.Qwen2VLProcessor`.
743 + #
744 + # The following code is equivalent to:
745 + # prompt = llm_inputs["prompt"]
746 + # inputs = processor(text=[prompt],
747 + # images=image_inputs,
748 + # videos=video_inputs,
749 + # padding=True,
750 + # return_tensors="pt")
751 + # prompt_token_ids = inputs["input_ids"][0].tolist()
752 +
753 + prompt_token_ids = llm_inputs.get("prompt_token_ids", None)
754 + if prompt_token_ids is None:
755 + prompt = llm_inputs["prompt"]
756 + prompt_token_ids = processor.tokenizer(
757 + prompt,
758 + padding=True,
759 + return_tensors=None,
760 + )["input_ids"]
761 +
762 + # Expand image pad tokens.
763 + if image_inputs is not None:
764 + image_indices = [
765 + idx for idx, token in enumerate(prompt_token_ids)
766 + if token == hf_config.image_token_id
767 + ]
768 + image_inputs = make_batched_images(image_inputs)
769 + assert len(image_indices) == len(image_inputs)
770 +
771 + prompt_token_ids_with_image = []
772 + for image_cnt, image in enumerate(image_inputs):
773 + num_image_tokens = _get_llm_num_vision_tokens(
774 + [image],
775 + data_type_key="image",
776 + image_processor=image_processor,
777 + )
778 + if image_cnt == 0:
779 + non_image_tokens = prompt_token_ids[:image_indices[image_cnt]]
780 + else:
781 + non_image_tokens = prompt_token_ids[image_indices[image_cnt -
782 + 1] +
783 + 1:image_indices[image_cnt]]
784 + prompt_token_ids_with_image.extend(non_image_tokens)
785 + prompt_token_ids_with_image.extend(
786 + hf_config.image_token_id for _ in range(num_image_tokens))
787 + prompt_token_ids_with_image.extend(prompt_token_ids[image_indices[-1] +
788 + 1:])
789 + prompt_token_ids = prompt_token_ids_with_image
790 +
791 + # Expand video pad tokens.
792 + if video_inputs is not None:
793 + video_indices = [
794 + idx for idx, token in enumerate(prompt_token_ids)
795 + if token == hf_config.video_token_id
796 + ]
797 + video_inputs = make_batched_videos(video_inputs)
798 + assert len(video_indices) == len(video_inputs)
799 +
800 + prompt_token_ids_with_video = []
801 + for video_cnt, video in enumerate(video_inputs):
802 + num_video_tokens = _get_llm_num_vision_tokens(
803 + video,
804 + data_type_key="video",
805 + image_processor=image_processor,
806 + )
807 + if video_cnt == 0:
808 + non_video_tokens = prompt_token_ids[:video_indices[video_cnt]]
809 + else:
810 + non_video_tokens = prompt_token_ids[video_indices[video_cnt -
811 + 1] +
812 + 1:video_indices[video_cnt]]
813 + prompt_token_ids_with_video.extend(non_video_tokens)
814 + prompt_token_ids_with_video.extend(
815 + hf_config.video_token_id for _ in range(num_video_tokens))
816 + prompt_token_ids_with_video.extend(prompt_token_ids[video_indices[-1] +
817 + 1:])
818 + prompt_token_ids = prompt_token_ids_with_video
819 +
820 + return LLMInputs(
821 + prompt_token_ids=prompt_token_ids,
822 + prompt=llm_inputs["prompt"],
823 + multi_modal_data=multi_modal_data,
824 + )
825 +
826 +
827 +@MULTIMODAL_REGISTRY.register_image_input_mapper(
828 + image_input_mapper_for_qwen2_vl)
829 +@MULTIMODAL_REGISTRY.register_input_mapper("video",
830 + video_input_mapper_for_qwen2_vl)
831 +@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_qwen2_vl_image_tokens)
832 +@MULTIMODAL_REGISTRY.register_max_multimodal_tokens(
833 + "video", get_max_qwen2_vl_video_tokens)
834 +@INPUT_REGISTRY.register_dummy_data(dummy_data_for_qwen2_vl)
835 +@INPUT_REGISTRY.register_input_processor(input_processor_for_qwen2_vl)
836 +class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal):
837 +
838 + def __init__(self,
839 + config: Qwen2VLConfig,
840 + multimodal_config: MultiModalConfig,
841 + cache_config: Optional[CacheConfig] = None,
842 + quant_config: Optional[QuantizationConfig] = None) -> None:
843 + super().__init__()
844 +
845 + assert not cache_config.enable_prefix_caching, \
846 + "Qwen2-VL currently does not support prefix caching"
847 +
848 + self.config = config
849 + self.multimodal_config = multimodal_config
850 +
851 + self.visual = Qwen2VisionTransformer(
852 + config.vision_config,
853 + norm_eps=getattr(config, "rms_norm_eps", 1e-6),
854 +
855 + # NOTE: Qwen2-VL vision encoder does not support any
856 + # quantization method now.
857 + quant_config=None,
858 + )
859 +
860 + self.model = Qwen2Model(config, cache_config, quant_config)
861 +
862 + if config.tie_word_embeddings:
863 + self.lm_head = self.model.embed_tokens
864 + else:
865 + self.lm_head = ParallelLMHead(config.vocab_size,
866 + config.hidden_size,
867 + quant_config=quant_config)
868 +
869 + self.logits_processor = LogitsProcessor(config.vocab_size)
870 + self.sampler = Sampler()
871 +
872 + def _validate_and_reshape_mm_tensor(self,
873 + mm_input: Union[torch.Tensor,
874 + List[torch.Tensor]],
875 + name: str) -> torch.Tensor:
876 + if not isinstance(mm_input, (torch.Tensor, list)):
877 + raise ValueError(f"Incorrect type of {name}. "
878 + f"Got type: {type(mm_input)}")
879 + if isinstance(mm_input, torch.Tensor):
880 + if mm_input.ndim == 2:
881 + return mm_input
882 + if mm_input.ndim != 3:
883 + raise ValueError(f"{name} should be 2D or batched 3D tensor. "
884 + f"Got ndim: {mm_input.ndim}")
885 + return torch.concat(list(mm_input))
886 + else:
887 + return torch.concat(mm_input)
888 +
889 + def _parse_and_validate_image_input(
890 + self, **kwargs: object) -> Optional[Qwen2VLImageInputs]:
891 + pixel_values = kwargs.pop("pixel_values", None)
892 + image_grid_thw = kwargs.pop("image_grid_thw", None)
893 +
894 + if pixel_values is None:
895 + return None
896 +
897 + pixel_values = self._validate_and_reshape_mm_tensor(
898 + pixel_values, "image pixel values")
899 + image_grid_thw = self._validate_and_reshape_mm_tensor(
900 + image_grid_thw, "image grid_thw")
901 +
902 + if not isinstance(pixel_values, (torch.Tensor, list)):
903 + raise ValueError("Incorrect type of image pixel values. "
904 + f"Got type: {type(pixel_values)}")
905 +
906 + return Qwen2VLImageInputs(pixel_values=pixel_values,
907 + image_grid_thw=image_grid_thw)
908 +
909 + def _parse_and_validate_video_input(
910 + self, **kwargs: object) -> Optional[Qwen2VLVideoInputs]:
911 + pixel_values_videos = kwargs.pop("pixel_values_videos", None)
912 + video_grid_thw = kwargs.pop("video_grid_thw", None)
913 +
914 + if pixel_values_videos is None:
915 + return None
916 +
917 + pixel_values_videos = self._validate_and_reshape_mm_tensor(
918 + pixel_values_videos, "video pixel values")
919 + video_grid_thw = self._validate_and_reshape_mm_tensor(
920 + video_grid_thw, "video grid_thw")
921 +
922 + return Qwen2VLVideoInputs(
923 + pixel_values_videos=pixel_values_videos,
924 + video_grid_thw=video_grid_thw,
925 + )
926 +
927 + def _process_image_input(self,
928 + image_input: Qwen2VLImageInputs) -> torch.Tensor:
929 + pixel_values = image_input["pixel_values"].type(self.visual.dtype)
930 + image_embeds = self.visual(pixel_values,
931 + grid_thw=image_input["image_grid_thw"])
932 + return image_embeds
933 +
934 + def _process_video_input(self,
935 + video_input: Qwen2VLVideoInputs) -> torch.Tensor:
936 + pixel_values_videos = video_input["pixel_values_videos"].type(
937 + self.visual.dtype)
938 + video_embeds = self.visual(pixel_values_videos,
939 + grid_thw=video_input["video_grid_thw"])
940 + return video_embeds
941 +
942 + def _merge_multimodal_embeddings(
943 + self,
944 + input_ids: torch.Tensor,
945 + inputs_embeds: torch.Tensor,
946 + multimodal_embeddings: torch.Tensor,
947 + placeholder_token_id: int,
948 + ) -> torch.Tensor:
949 + mask = (input_ids == placeholder_token_id)
950 + inputs_embeds[mask, :] = multimodal_embeddings
951 + return inputs_embeds
952 +
953 + def forward(
954 + self,
955 + input_ids: torch.Tensor,
956 + positions: torch.Tensor,
957 + kv_caches: List[torch.Tensor],
958 + attn_metadata: AttentionMetadata,
959 + intermediate_tensors: Optional[IntermediateTensors] = None,
960 + **kwargs: object,
961 + ) -> SamplerOutput:
962 + """Run forward pass for Qwen2-VL.
963 +
964 + Args:
965 + input_ids: Flattened (concatenated) input_ids corresponding to a
966 + batch.
967 + positions: Flattened (concatenated) position ids corresponding to a
968 + batch.
969 + **NOTE**: If mrope is enabled (default setting for Qwen2-VL
970 + opensource models), the shape will be `(3, seq_len)`,
971 + otherwise it will be `(seq_len,).
972 + pixel_values: Pixel values to be fed to a model.
973 + `None` if no images are passed.
974 + image_grid_thw: Tensor `(n_images, 3)` of image 3D grid in LLM.
975 + `None` if no images are passed.
976 + pixel_values_videos: Pixel values of videos to be fed to a model.
977 + `None` if no videos are passed.
978 + video_grid_thw: Tensor `(n_videos, 3)` of video 3D grid in LLM.
979 + `None` if no videos are passed.
980 + """
981 +
982 + image_input = self._parse_and_validate_image_input(**kwargs)
983 + video_input = self._parse_and_validate_video_input(**kwargs)
984 +
985 + if image_input is None and video_input is None:
986 + inputs_embeds = None
987 + else:
988 + if getattr(self.config, "rope_scaling", {}).get("type",
989 + None) == "mrope":
990 + assert positions.ndim == 2 and positions.size(0) == 3, (
991 + "multimodal section rotary embedding requires "
992 + f"(3, seq_len) positions, but got {positions.size()}")
993 +
994 + inputs_embeds = self.model.embed_tokens(input_ids)
995 +
996 + if image_input is not None:
997 + image_embeds = self._process_image_input(image_input)
998 + inputs_embeds = self._merge_multimodal_embeddings(
999 + input_ids,
1000 + inputs_embeds,
1001 + image_embeds,
1002 + placeholder_token_id=self.config.image_token_id,
1003 + )
1004 +
1005 + if video_input is not None:
1006 + video_embeds = self._process_video_input(video_input)
1007 + inputs_embeds = self._merge_multimodal_embeddings(
1008 + input_ids,
1009 + inputs_embeds,
1010 + video_embeds,
1011 + placeholder_token_id=self.config.video_token_id,
1012 + )
1013 +
1014 + input_ids = None
1015 +
1016 + hidden_states = self.model(
1017 + input_ids=input_ids,
1018 + positions=positions,
1019 + kv_caches=kv_caches,
1020 + attn_metadata=attn_metadata,
1021 + inputs_embeds=inputs_embeds,
1022 + )
1023 + return hidden_states
1024 +
1025 + def compute_logits(self, hidden_states: torch.Tensor,
1026 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
1027 + logits = self.logits_processor(self.lm_head, hidden_states,
1028 + sampling_metadata)
1029 + return logits
1030 +
1031 + def sample(
1032 + self,
1033 + logits: torch.Tensor,
1034 + sampling_metadata: SamplingMetadata,
1035 + ) -> Optional[SamplerOutput]:
1036 + next_tokens = self.sampler(logits, sampling_metadata)
1037 + return next_tokens
1038 +
1039 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
1040 + stacked_params_mapping = [
1041 + # (param_name, shard_name, shard_id)
1042 + ("qkv_proj", "q_proj", "q"),
1043 + ("qkv_proj", "k_proj", "k"),
1044 + ("qkv_proj", "v_proj", "v"),
1045 + ("gate_up_proj", "up_proj", 1),
1046 + ("gate_up_proj", "gate_proj", 0),
1047 + ]
1048 + params_dict = dict(self.named_parameters(remove_duplicate=False))
1049 + for name, loaded_weight in weights:
1050 + if "rotary_emb.inv_freq" in name:
1051 + continue
1052 + if self.config.tie_word_embeddings and "lm_head.weight" in name:
1053 + continue
1054 + for (param_name, weight_name, shard_id) in stacked_params_mapping:
1055 + if weight_name not in name:
1056 + continue
1057 + name = name.replace(weight_name, param_name)
1058 + param = params_dict[name]
1059 + weight_loader = param.weight_loader
1060 + weight_loader(param, loaded_weight, shard_id)
1061 + break
1062 + else:
1063 + if "visual" in name and "qkv.weight" in name:
1064 + visual_num_heads = self.config.vision_config.num_heads
1065 + visual_embed_dim = self.config.vision_config.embed_dim
1066 + head_size = visual_embed_dim // visual_num_heads
1067 + loaded_weight = loaded_weight.view(3, visual_num_heads,
1068 + head_size,
1069 + visual_embed_dim)
1070 + loaded_weight = loaded_weight.transpose(0, 1)
1071 + loaded_weight = loaded_weight.reshape(-1, visual_embed_dim)
1072 + elif "visual" in name and "qkv.bias" in name:
1073 + visual_num_heads = self.config.vision_config.num_heads
1074 + visual_embed_dim = self.config.vision_config.embed_dim
1075 + head_size = visual_embed_dim // visual_num_heads
1076 + loaded_weight = loaded_weight.view(3, visual_num_heads,
1077 + head_size)
1078 + loaded_weight = loaded_weight.transpose(0, 1)
1079 + loaded_weight = loaded_weight.reshape(-1)
1080 + try:
1081 + param = params_dict[name]
1082 + except KeyError:
1083 + print(params_dict.keys())
1084 + raise
1085 +
1086 + weight_loader = getattr(param, "weight_loader",
1087 + default_weight_loader)
1088 + weight_loader(param, loaded_weight)
vllm/multimodal/base.py
+3 -5
@@ -79,14 +79,12 @@ class MultiModalInputs(_MultiModalInputsBase):
79 if len(inputs_list) == 0:
80 return {}
81
82 - keys = inputs_list[0].keys()
83 -
82 item_lists: Dict[str, List[NestedTensors]] = defaultdict(list)
83
84 for inputs in inputs_list:
87 - if inputs.keys() != keys:
88 - msg = f"Inputs do not share the same keys ({keys})"
89 - raise ValueError(msg)
85 + # For models that supports multiple modalities (e.g. Qwen2-VL),
86 + # different modalities will return different data keys,
87 + # so batch() should skip the same key check.
88
89 for k, v in inputs.items():
90 item_lists[k].append(v)
vllm/sequence.py
+11
@@ -165,6 +165,9 @@ class SequenceData(msgspec.Struct,
165 # is called.
166 _new_appended_tokens: List[int] = msgspec.field(default_factory=list)
167
168 + # It is used to compute mrope_position_ids.
169 + _mrope_position_delta: Optional[int] = None
170 +
171 def __post_init__(self) -> None:
172 assert self._prompt_token_ids.typecode == "l"
173 assert self._output_token_ids.typecode == "l"
@@ -219,6 +222,14 @@ class SequenceData(msgspec.Struct,
222 assert isinstance(self._output_token_ids, array)
223 return self._output_token_ids
224
225 + @property
226 + def mrope_position_delta(self) -> Optional[int]:
227 + return self._mrope_position_delta
228 +
229 + @mrope_position_delta.setter
230 + def mrope_position_delta(self, new_mrope_position_delta):
231 + self._mrope_position_delta = new_mrope_position_delta
232 +
233 def append_token_id(self, token_id: int, logprob: float) -> None:
234 self._output_token_ids.append(token_id)
235 self._new_appended_tokens.append(token_id)
vllm/transformers_utils/processor.py new
+37
@@ -0,0 +1,37 @@
1 +from typing import cast
2 +
3 +
4 +def get_processor(
5 + processor_name: str,
6 + *args,
7 + trust_remote_code: bool = False,
8 + **kwargs,
9 +):
10 + """Gets a processor for the given model name via HuggingFace."""
11 + # don't put this import at the top level
12 + # it will call torch.cuda.device_count()
13 + from transformers import AutoProcessor
14 + from transformers.processing_utils import ProcessorMixin
15 +
16 + try:
17 + processor = AutoProcessor.from_pretrained(
18 + processor_name,
19 + *args,
20 + trust_remote_code=trust_remote_code,
21 + **kwargs)
22 + except ValueError as e:
23 + # If the error pertains to the processor class not existing or not
24 + # currently being imported, suggest using the --trust-remote-code flag.
25 + # Unlike AutoTokenizer, AutoProcessor does not separate such errors
26 + if not trust_remote_code:
27 + err_msg = (
28 + "Failed to load the processor. If the processor is "
29 + "a custom processor not yet available in the HuggingFace "
30 + "transformers library, consider setting "
31 + "`trust_remote_code=True` in LLM or using the "
32 + "`--trust-remote-code` flag in the CLI.")
33 + raise RuntimeError(err_msg) from e
34 + else:
35 + raise e
36 +
37 + return cast(ProcessorMixin, processor)
vllm/worker/model_runner.py
+99 -11
@@ -30,6 +30,7 @@ from vllm.lora.layers import LoRAMapping
30 from vllm.lora.request import LoRARequest
31 from vllm.lora.worker_manager import LRUCacheWorkerLoRAManager
32 from vllm.model_executor import SamplingMetadata, SamplingMetadataCache
33 +from vllm.model_executor.layers.rotary_embedding import MRotaryEmbedding
34 from vllm.model_executor.layers.sampler import SamplerOutput
35 from vllm.model_executor.model_loader import get_model
36 from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
@@ -181,6 +182,7 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
182 def simple_reinit(self):
183 self.input_tokens[0].clear() # type: ignore
184 self.input_positions[0].clear() # type: ignore
185 + self.mrope_input_positions = None # type: ignore
186 self.seq_lens[0] = 0 # type: ignore
187 self.orig_seq_lens[0] = 0 # type: ignore
188 self.query_lens[0] = 0 # type: ignore
@@ -206,6 +208,7 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
208 # Input tokens and positions.
209 input_tokens: Optional[List[List[int]]] = None,
210 input_positions: Optional[List[List[int]]] = None,
211 + mrope_input_positions: Optional[List[List[List[int]]]] = None,
212
213 # The sequence length (may be capped to the sliding window).
214 seq_lens: Optional[List[int]] = None,
@@ -266,6 +269,8 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
269 for seq_id in range(len(self.seq_ids)):
270 self.input_positions[seq_id].clear()
271
272 + self.mrope_input_positions = None
273 +
274 if seq_lens:
275 self.seq_lens = seq_lens
276 else:
@@ -327,6 +332,7 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
332 else:
333 self.input_tokens = input_tokens or []
334 self.input_positions = input_positions or []
335 + self.mrope_input_positions = mrope_input_positions or None
336 self.seq_lens = seq_lens or []
337 self.orig_seq_lens = orig_seq_lens or []
338 self.query_lens = query_lens or []
@@ -357,6 +363,7 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
363
364 self.input_tokens = [[] for _ in range(self.n_seqs)]
365 self.input_positions = [[] for _ in range(self.n_seqs)]
366 + self.mrope_input_positions = None
367 self.seq_lens = [0] * self.n_seqs
368 self.orig_seq_lens = [0] * self.n_seqs
369 self.query_lens = [0] * self.n_seqs
@@ -493,6 +500,17 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
500 inter_data.query_lens[
501 seq_idx] = seq_len - context_len if inter_data.is_prompt else 1
502
503 + if seq_data.mrope_position_delta is not None:
504 + if inter_data.mrope_input_positions is None:
505 + inter_data.mrope_input_positions = [None] * inter_data.n_seqs
506 +
507 + inter_data.mrope_input_positions[
508 + seq_idx] = MRotaryEmbedding.get_next_input_positions(
509 + seq_data.mrope_position_delta,
510 + context_len,
511 + seq_len,
512 + )
513 +
514 def _compute_for_prefix_cache_hit(
515 self, inter_data: InterDataForSeqGroup, seq_idx: int,
516 seq_group_metadata: SequenceGroupMetadata):
@@ -636,6 +654,40 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
654 mm_kwargs = self.multi_modal_input_mapper(mm_data)
655 inter_data.multi_modal_inputs = mm_kwargs
656
657 + # special processing for mrope position deltas.
658 + if self.runner.model_is_mrope:
659 + image_grid_thw = mm_kwargs.get("image_grid_thw", None)
660 + video_grid_thw = mm_kwargs.get("video_grid_thw", None)
661 + assert image_grid_thw is not None or video_grid_thw is not None, (
662 + "mrope embedding type requires multi-modal input mapper "
663 + "returns 'image_grid_thw' or 'video_grid_thw'.")
664 +
665 + hf_config = self.runner.model_config.hf_config
666 +
667 + inter_data.mrope_input_positions = [None] * inter_data.n_seqs
668 + for seq_idx in range(inter_data.n_seqs):
669 + seq_data = seq_group_metadata.seq_data[
670 + inter_data.seq_ids[seq_idx]]
671 + token_ids = seq_data.get_token_ids()
672 +
673 + mrope_input_positions, mrope_position_delta = \
674 + MRotaryEmbedding.get_input_positions(
675 + token_ids,
676 + image_grid_thw=image_grid_thw,
677 + video_grid_thw=video_grid_thw,
678 + image_token_id=hf_config.image_token_id,
679 + video_token_id=hf_config.video_token_id,
680 + vision_start_token_id=hf_config.vision_start_token_id,
681 + vision_end_token_id=hf_config.vision_end_token_id,
682 + spatial_merge_size=hf_config.vision_config.
683 + spatial_merge_size,
684 + context_len=inter_data.context_lens[seq_idx],
685 + )
686 +
687 + seq_data.mrope_position_delta = mrope_position_delta
688 + inter_data.mrope_input_positions[
689 + seq_idx] = mrope_input_positions
690 +
691 def add_seq_group(self, seq_group_metadata: SequenceGroupMetadata):
692 """Add a sequence group to the builder."""
693 seq_ids = seq_group_metadata.seq_data.keys()
@@ -684,10 +736,27 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
736 # prefix caching and there is no decode request.
737 return self.model_input_cls()
738
687 - input_positions = []
688 - for inter_data in self.inter_data_list:
689 - for cur_input_positions in inter_data.input_positions:
690 - input_positions.extend(cur_input_positions)
739 + mrope_input_positions: Optional[List[List[int]]] = None
740 + if any(inter_data.mrope_input_positions is not None
741 + for inter_data in self.inter_data_list):
742 + mrope_input_positions = [[] for _ in range(3)]
743 + for idx in range(3):
744 + for inter_data in self.inter_data_list:
745 + msections = inter_data.mrope_input_positions
746 + if msections is None:
747 + for _seq_input_positions in inter_data.input_positions:
748 + mrope_input_positions[idx].extend(
749 + _seq_input_positions)
750 + else:
751 + for _seq_mrope_input_positions in msections:
752 + mrope_input_positions[idx].extend(
753 + _seq_mrope_input_positions[idx])
754 + input_positions = None
755 + else:
756 + input_positions = []
757 + for inter_data in self.inter_data_list:
758 + for cur_input_positions in inter_data.input_positions:
759 + input_positions.extend(cur_input_positions)
760
761 seq_lens = []
762 max_decode_seq_len = 0
@@ -724,15 +793,24 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
793 # Tokens and positions.
794 if cuda_graph_pad_size:
795 input_tokens.extend(itertools.repeat(0, cuda_graph_pad_size))
727 - input_positions.extend(itertools.repeat(0, cuda_graph_pad_size))
796 assert self.runner.device is not None
797 input_tokens_tensor = async_tensor_h2d(input_tokens, torch.long,
798 self.runner.device,
799 self.runner.pin_memory)
732 - input_positions_tensor = async_tensor_h2d(input_positions, torch.long,
733 - self.runner.device,
734 - self.runner.pin_memory)
735 -
800 + if mrope_input_positions is not None:
801 + for idx in range(3):
802 + mrope_input_positions[idx].extend(
803 + itertools.repeat(0, cuda_graph_pad_size))
804 + input_positions_tensor = async_tensor_h2d(mrope_input_positions,
805 + torch.long,
806 + self.runner.device,
807 + self.runner.pin_memory)
808 + else:
809 + input_positions.extend(itertools.repeat(0, cuda_graph_pad_size))
810 + input_positions_tensor = async_tensor_h2d(input_positions,
811 + torch.long,
812 + self.runner.device,
813 + self.runner.pin_memory)
814 # Sequence and query lengths.
815 if cuda_graph_pad_size:
816 seq_lens.extend(itertools.repeat(1, cuda_graph_pad_size))
@@ -1199,6 +1277,15 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1277 raise RuntimeError("PromptAdapter is not enabled.")
1278 return self.prompt_adapter_manager.list_adapters()
1279
1280 + @property
1281 + def model_is_mrope(self) -> bool:
1282 + """Detect if the model has "mrope" rope_scaling type.
1283 + mrope requires keep "rope_deltas" between prompt and decoding phases."""
1284 + rope_scaling = getattr(self.model_config.hf_config, "rope_scaling", {})
1285 + if rope_scaling is None:
1286 + return False
1287 + return rope_scaling.get("type", None) == "mrope"
1288 +
1289 @torch.inference_mode()
1290 def capture_model(self, kv_caches: List[List[torch.Tensor]]) -> None:
1291 """Cuda graph capture a model.
@@ -1229,7 +1316,8 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1316 max_batch_size = self.max_batchsize_to_capture
1317 input_tokens = torch.zeros(max_batch_size, dtype=torch.long).cuda()
1318 input_positions = torch.zeros(max_batch_size, dtype=torch.long).cuda()
1232 -
1319 + if self.model_is_mrope:
1320 + input_positions = torch.tile(input_positions, (3, 1))
1321 # Prepare dummy previous_hidden_states only if needed by the model.
1322 # This is used by draft models such as EAGLE.
1323 previous_hidden_states = None
@@ -1293,7 +1381,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1381 "input_ids":
1382 input_tokens[:batch_size],
1383 "positions":
1296 - input_positions[:batch_size],
1384 + input_positions[..., :batch_size],
1385 "hidden_or_intermediate_states":
1386 hidden_or_intermediate_states[
1387 virtual_engine] # type: ignore