A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[MODEL] Qwen Multimodal Support (Qwen-VL / Qwen-VL-Chat) (#8029)
Signed-off-by: Alex-Brooks <Alex.Brooks@ibm.com> Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>
Alex Brooks committed
Sep 5, 2024 at 06:48 UTC
9da25a88aa35da4b5ad7da545e6189e08c5f52f4
8 files changed
+1110
-208
docs/source/models/supported_models.rst
+5
@@ -242,6 +242,11 @@ Multimodal Language Models
242
- Image\ :sup:`+`
243
- :code:`openbmb/MiniCPM-V-2` (see note), :code:`openbmb/MiniCPM-Llama3-V-2_5`, :code:`openbmb/MiniCPM-V-2_6`, etc.
244
-
245
+ * - :code:`QWenLMHeadModel`
246
+ - Qwen
247
+ - Image
248
+ - :code:`Qwen/Qwen-VL`, :code:`Qwen/Qwen-VL-Chat`, etc.
249
+ -
250
* - :code:`UltravoxModel`
251
- Ultravox
252
- Audio\ :sup:`E+`
examples/offline_inference_vision_language.py
+15
@@ -159,6 +159,20 @@ def run_blip2(question):
159
return llm, prompt, stop_token_ids
160
161
162
+# Qwen
163
+def run_qwen_vl(question):
164
+
165
+ llm = LLM(
166
+ model="Qwen/Qwen-VL",
167
+ trust_remote_code=True,
168
+ max_num_seqs=5,
169
+ )
170
+
171
+ prompt = f"{question}Picture 1: <img></img>\n"
172
+ stop_token_ids = None
173
+ return llm, prompt, stop_token_ids
174
+
175
+
176
model_example_map = {
177
"llava": run_llava,
178
"llava-next": run_llava_next,
@@ -169,6 +183,7 @@ model_example_map = {
183
"minicpmv": run_minicpmv,
184
"blip-2": run_blip2,
185
"internvl_chat": run_internvl,
186
+ "qwen_vl": run_qwen_vl,
187
}
188
189
tests/models/test_qwen.py
+142
-25
@@ -1,48 +1,165 @@
1
-from typing import Type
1
+import pathlib
2
+from typing import List, Optional, Type
3
4
import pytest
5
5
-from ..conftest import HfRunner, VllmRunner
6
+from vllm.multimodal.utils import rescale_image_size
7
+
8
+from ..conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
9
from .utils import check_logprobs_close
10
8
-models = ["qwen/qwen-vl"]
11
+pytestmark = pytest.mark.vlm
12
13
+text_only_models = [
14
+ "Qwen/Qwen-7B-Chat" # Has no visual component
15
+]
16
11
-@pytest.mark.parametrize("dtype", ["half"])
12
-@pytest.mark.parametrize("max_tokens", [32])
13
-@pytest.mark.parametrize("num_logprobs", [5])
14
-@pytest.mark.parametrize("model", models)
15
-def test_text_only_qwen_model(
17
+multimodal_models = ["Qwen/Qwen-VL"]
18
+
19
+HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
20
+ "stop_sign":
21
+ "Picture 1: <img></img>\nWhat's the content of the image?: ",
22
+ "cherry_blossom":
23
+ "Picture 1: <img></img>\nWhat is the season?: ",
24
+})
25
+
26
+
27
+### Tests for multimodal Qwen models
28
+def run_test(
29
+ tmp_path: pathlib.PosixPath,
30
hf_runner: Type[HfRunner],
31
vllm_runner: Type[VllmRunner],
18
- example_prompts,
32
+ image_assets: _ImageAssets,
33
model: str,
34
*,
35
+ size_factors: List[float],
36
dtype: str,
37
max_tokens: int,
38
num_logprobs: int,
39
+ tensor_parallel_size: int,
40
+ distributed_executor_backend: Optional[str] = None,
41
):
25
- # This test checks language inputs only, since the visual component
26
- # for qwen-vl is still unsupported in VLLM. In the near-future, the
27
- # implementation and this test will be extended to consider
28
- # visual inputs as well.
42
+ """Inference result should be the same between hf and vllm.
43
+
44
+ All the image fixtures for the test is under tests/images.
45
+ For huggingface runner, we provide the PIL images as input.
46
+ For vllm runner, we provide MultiModalDataDict objects
47
+ and corresponding MultiModalConfig as input.
48
+ Note, the text input is also adjusted to abide by vllm contract.
49
+ The text output is sanitized to be able to compare with hf.
50
+ """
51
+ images = [asset.pil_image for asset in image_assets]
52
+
53
+ # Export the images to a tempdir and substitute it into the hf prompt;
54
+ # the contents between <img>/</img> will be ignored by VLLM, but the
55
+ # transformers implementation for the visual transformer parses this to
56
+ # reload it in the forward call; the contents are treated as a URL or a
57
+ # local path.
58
+ for idx, asset in enumerate(image_assets):
59
+ image_tmp_path = tmp_path / f"{asset.name}.jpg"
60
+ asset.pil_image.save(image_tmp_path)
61
+ HF_IMAGE_PROMPTS[idx] = HF_IMAGE_PROMPTS[idx].replace(
62
+ "<img></img>", f"<img>{image_tmp_path}</img>")
63
+
64
+ inputs_per_image = [(
65
+ [prompt for _ in size_factors],
66
+ [rescale_image_size(image, factor) for factor in size_factors],
67
+ ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
68
+
69
+ # NOTE: take care of the order. run vLLM first, and then run HF.
70
+ # vLLM needs a fresh new process without cuda initialization.
71
+ # if we run HF first, the cuda initialization will be done and it
72
+ # will hurt multiprocessing backend with fork method (the default method).
73
+
74
+ # max_model_len should be greater than image_feature_size
75
+ # Qwen encodes images into a fixed content size of 256
76
+ with vllm_runner(model,
77
+ max_model_len=300,
78
+ max_num_seqs=1,
79
+ dtype=dtype,
80
+ tensor_parallel_size=tensor_parallel_size,
81
+ distributed_executor_backend=distributed_executor_backend,
82
+ enforce_eager=True) as vllm_model:
83
+ vllm_outputs_per_image = [
84
+ vllm_model.generate_greedy_logprobs(prompts,
85
+ max_tokens,
86
+ num_logprobs=num_logprobs,
87
+ images=images)
88
+ for prompts, images in inputs_per_image
89
+ ]
90
+
91
with hf_runner(model, dtype=dtype) as hf_model:
30
- hf_outputs = hf_model.generate_greedy_logprobs_limit(
31
- example_prompts,
32
- max_tokens,
33
- num_logprobs=num_logprobs,
92
+ hf_outputs_per_image = [
93
+ hf_model.generate_greedy_logprobs_limit(prompts,
94
+ max_tokens,
95
+ num_logprobs=num_logprobs,
96
+ images=images)
97
+ for prompts, images in inputs_per_image
98
+ ]
99
+
100
+ for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
101
+ vllm_outputs_per_image):
102
+
103
+ check_logprobs_close(
104
+ outputs_0_lst=hf_outputs,
105
+ outputs_1_lst=vllm_outputs,
106
+ name_0="hf",
107
+ name_1="vllm",
108
)
109
110
+
111
+@pytest.mark.parametrize("model", multimodal_models)
112
+@pytest.mark.parametrize(
113
+ "size_factors",
114
+ [
115
+ # No image
116
+ [],
117
+ # Single-scale
118
+ [1.0],
119
+ # Single-scale, batched
120
+ [1.0, 1.0, 1.0],
121
+ # Multi-scale
122
+ [0.25, 0.5, 1.0],
123
+ ],
124
+)
125
+@pytest.mark.parametrize("dtype", ["bfloat16"])
126
+@pytest.mark.parametrize("max_tokens", [8])
127
+@pytest.mark.parametrize("num_logprobs", [5])
128
+def test_multimodal_models(tmp_path, hf_runner, vllm_runner, image_assets,
129
+ model, size_factors, dtype, max_tokens,
130
+ num_logprobs) -> None:
131
+ run_test(
132
+ tmp_path,
133
+ hf_runner,
134
+ vllm_runner,
135
+ image_assets,
136
+ model,
137
+ size_factors=size_factors,
138
+ dtype=dtype,
139
+ max_tokens=max_tokens,
140
+ num_logprobs=num_logprobs,
141
+ tensor_parallel_size=1,
142
+ )
143
+
144
+
145
+# Ensure that a text-only Qwen model can still be loaded and
146
+# used for inference in VLLM without throwing.
147
+@pytest.mark.parametrize("model", text_only_models)
148
+@pytest.mark.parametrize("dtype", ["bfloat16"])
149
+@pytest.mark.parametrize("max_tokens", [32])
150
+@pytest.mark.parametrize("num_logprobs", [5])
151
+def test_text_only_qwen_model_can_be_loaded_and_run(
152
+ vllm_runner: Type[VllmRunner],
153
+ example_prompts,
154
+ model: str,
155
+ *,
156
+ dtype: str,
157
+ max_tokens: int,
158
+ num_logprobs: int,
159
+):
160
with vllm_runner(model, dtype=dtype) as vllm_model:
37
- vllm_outputs = vllm_model.generate_greedy_logprobs(
161
+ vllm_model.generate_greedy_logprobs(
162
example_prompts,
163
max_tokens,
164
num_logprobs=num_logprobs,
165
)
42
-
43
- check_logprobs_close(
44
- outputs_0_lst=hf_outputs,
45
- outputs_1_lst=vllm_outputs,
46
- name_0="hf",
47
- name_1="vllm",
48
- )
vllm/entrypoints/chat_utils.py
+2
@@ -150,6 +150,8 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
150
if model_type in ("blip-2", "chatglm", "fuyu", "paligemma"):
151
# These models do not use image tokens in the prompt
152
return None
153
+ if model_type == "qwen":
154
+ return f"Picture {current_count}: <img></img>"
155
if model_type.startswith("llava"):
156
return self._cached_token_str(self._tokenizer,
157
hf_config.image_token_index)
vllm/model_executor/layers/resampler.py
new
+273
@@ -0,0 +1,273 @@
1
+# coding=utf-8
2
+# Adapted from
3
+# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
4
+# https://huggingface.co/Qwen/Qwen-7B/blob/main/modeling_qwen.py
5
+# https://github.com/facebookresearch/mae/blob/efb2a8062c206524e35e47d04501ed4f544c0ae8/util/pos_embed.py#L20
6
+#
7
+# Copyright 2023 The Qwen team.
8
+# Copyright 2023 The vLLM team.
9
+# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
10
+#
11
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
12
+# and OPT implementations in this library. It has been modified from its
13
+# original forms to accommodate minor architectural differences compared
14
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
15
+#
16
+# Licensed under the Apache License, Version 2.0 (the "License");
17
+# you may not use this file except in compliance with the License.
18
+# You may obtain a copy of the License at
19
+#
20
+# http://www.apache.org/licenses/LICENSE-2.0
21
+#
22
+# Unless required by applicable law or agreed to in writing, software
23
+# distributed under the License is distributed on an "AS IS" BASIS,
24
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
25
+# See the License for the specific language governing permissions and
26
+# limitations under the License.
27
+"""
28
+Shared resampler perceiver network used in multimodal models and
29
+related helpers for sincos positional embeddings.
30
+
31
+Example models: Qwen (Qwen-VL), Minicpmv2.0
32
+"""
33
+import math
34
+from functools import partial
35
+from typing import Callable, Optional, Tuple, Union
36
+
37
+import numpy as np
38
+import torch
39
+import torch.nn.functional as F
40
+from torch import nn
41
+from torch.nn.init import trunc_normal_
42
+
43
+from vllm.model_executor.layers.linear import ReplicatedLinear
44
+
45
+DEFAULT_LN = partial(nn.LayerNorm, eps=1e-6)
46
+
47
+
48
+def get_abs_pos(abs_pos: torch.Tensor, tgt_size: Union[torch.Tensor,
49
+ int]) -> torch.Tensor:
50
+ # abs_pos: L, C
51
+ # tgt_size: (H, W)
52
+ # return: M, C
53
+ src_size = int(math.sqrt(abs_pos.size(0)))
54
+ dtype = abs_pos.dtype
55
+ if isinstance(tgt_size, int):
56
+ tgt_size = (tgt_size, tgt_size)
57
+ if (src_size == tgt_size[0] and src_size == tgt_size[1]):
58
+ return abs_pos
59
+ return (F.interpolate(
60
+ abs_pos.float().reshape(1, src_size, src_size, -1).permute(0, 3, 1, 2),
61
+ size=(tgt_size[0], tgt_size[1]),
62
+ mode="bicubic",
63
+ align_corners=False,
64
+ ).permute(0, 2, 3, 1).flatten(0, 2).to(dtype=dtype))
65
+
66
+
67
+# sin/cos positional embedding helpers are adapted from:
68
+# https://github.com/facebookresearch/mae/blob/efb2a8062c206524e35e47d04501ed4f544c0ae8/util/pos_embed.py#L20
69
+def get_1d_sincos_pos_embed_from_grid(
70
+ embed_dim: int, pos: np.ndarray,
71
+ version: Tuple[int, int] = (2, 0)) -> torch.Tensor:
72
+ """
73
+ embed_dim: output dimension for each position
74
+ pos: a list of positions to be encoded: size (M,) / (H, W)
75
+ out: (M, D) / (H, W, D)
76
+ """
77
+ assert embed_dim % 2 == 0
78
+ omega = np.arange(embed_dim // 2, dtype=np.float32)
79
+ omega /= embed_dim / 2.0
80
+ omega = 1.0 / 10000**omega # (D/2,)
81
+
82
+ if version == (2, 0):
83
+ pos = pos.reshape(-1) # (M,)
84
+ out = np.einsum("m,d->md", pos, omega) # (M, D/2), outer product
85
+ emb_sin = np.sin(out) # (M, D/2)
86
+ emb_cos = np.cos(out) # (M, D/2)
87
+ emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D)
88
+ else:
89
+ out = np.einsum("hw,d->hwd", pos, omega) # (H, W, D/2), outer product
90
+ emb_sin = np.sin(out) # (H, W, D/2)
91
+ emb_cos = np.cos(out) # (H, W, D/2)
92
+ emb = np.concatenate([emb_sin, emb_cos], axis=-1) # (H, W, D)
93
+ return emb
94
+
95
+
96
+def get_2d_sincos_pos_embed_from_grid(
97
+ embed_dim: int, grid: np.ndarray,
98
+ version: Tuple[int, int] = (2, 0)) -> torch.Tensor:
99
+ assert embed_dim % 2 == 0
100
+
101
+ # use half of dimensions to encode grid_h
102
+ emb_h = get_1d_sincos_pos_embed_from_grid(
103
+ embed_dim // 2, grid[0], version) # (H*W, D/2) or (H, W, D/2)
104
+ emb_w = get_1d_sincos_pos_embed_from_grid(
105
+ embed_dim // 2, grid[1], version) # (H*W, D/2) or (H, W, D/2)
106
+
107
+ if version == (2, 0):
108
+ emb = np.concatenate([emb_h, emb_w], axis=1) # (H*W, D)
109
+ else:
110
+ emb = np.concatenate([emb_h, emb_w], axis=-1) # (H, W, D)
111
+ return emb
112
+
113
+
114
+def get_2d_sincos_pos_embed(
115
+ embed_dim: int,
116
+ grid_size: Union[int, Tuple[int, int]],
117
+ cls_token: bool = False,
118
+ version: Tuple[int, int] = (2, 0),
119
+) -> torch.Tensor:
120
+ """
121
+ grid_size: int of the grid height and width
122
+ return:
123
+ pos_embed: [grid_size*grid_size, embed_dim] or
124
+ [1+grid_size*grid_size, embed_dim] (w/ or w/o cls_token)
125
+ """
126
+ if isinstance(grid_size, int):
127
+ grid_h_size, grid_w_size = grid_size, grid_size
128
+ else:
129
+ grid_h_size, grid_w_size = grid_size[0], grid_size[1]
130
+
131
+ grid_h = np.arange(grid_h_size, dtype=np.float32)
132
+ grid_w = np.arange(grid_w_size, dtype=np.float32)
133
+ grid = np.meshgrid(grid_w, grid_h) # here w goes first
134
+ grid = np.stack(grid, axis=0)
135
+ assert isinstance(grid, np.ndarray) and \
136
+ grid.shape == (2, grid_h_size, grid_w_size)
137
+
138
+ if version == (2, 0):
139
+ grid = grid.reshape([2, 1, grid_h_size, grid_w_size])
140
+ pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid, version)
141
+ if cls_token:
142
+ pos_embed = np.concatenate([np.zeros([1, embed_dim]), pos_embed],
143
+ axis=0)
144
+ else:
145
+ pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid, version)
146
+ return pos_embed
147
+
148
+
149
+class BaseResampler(nn.Module):
150
+ """
151
+ A 2D perceiver-resampler network with one cross attention layers by
152
+ (grid_size**2) learnable queries and 2d sincos pos_emb.
153
+ Outputs:
154
+ A tensor with the shape of (grid_size**2, embed_dim)
155
+ """
156
+
157
+ def __init__(
158
+ self,
159
+ num_queries: int,
160
+ embed_dim: int,
161
+ num_heads: int,
162
+ kv_dim: Optional[int] = None,
163
+ norm_layer: Callable[[int], nn.LayerNorm] = DEFAULT_LN,
164
+ do_post_projection: bool = True,
165
+ ) -> None:
166
+ super().__init__()
167
+
168
+ self.num_queries = num_queries
169
+ self.embed_dim = embed_dim
170
+ self.num_heads = num_heads
171
+
172
+ self.query = nn.Parameter(torch.zeros(self.num_queries, embed_dim))
173
+ trunc_normal_(self.query, std=0.02)
174
+ if kv_dim is not None and kv_dim != embed_dim:
175
+ self.kv_proj = ReplicatedLinear(kv_dim, embed_dim, bias=False)
176
+ else:
177
+ # Maintain the same return value with ReplicatedLinear.forward
178
+ self.kv_proj = lambda *args, **kwargs: ( # type: ignore # noqa
179
+ nn.Identity()(*args, **kwargs),
180
+ None,
181
+ )
182
+ self.attn = nn.MultiheadAttention(embed_dim, num_heads)
183
+ self.ln_q = norm_layer(embed_dim)
184
+ self.ln_kv = norm_layer(embed_dim)
185
+ self.do_post_projection = do_post_projection
186
+ self.ln_post = norm_layer(embed_dim) if do_post_projection else None
187
+ self.proj = nn.Parameter(
188
+ (embed_dim**-0.5) *
189
+ torch.randn(embed_dim, embed_dim)) if do_post_projection else None
190
+
191
+ def _init_weights(self, m: nn.Module) -> None:
192
+ if isinstance(m, nn.Linear):
193
+ trunc_normal_(m.weight, std=0.02)
194
+ if isinstance(m, nn.Linear) and m.bias is not None:
195
+ nn.init.constant_(m.bias, 0)
196
+ elif isinstance(m, nn.LayerNorm):
197
+ nn.init.constant_(m.bias, 0)
198
+ nn.init.constant_(m.weight, 1.0)
199
+
200
+ def _repeat(self, query, N: int):
201
+ return query.unsqueeze(1).repeat(1, N, 1)
202
+
203
+
204
+class Resampler2(BaseResampler):
205
+ """Resampler-perceiver network to be used for a variety of model types,
206
+ e.g., Qwen-vl / Minicpmv 2.0. The main difference is the addition of the
207
+ do_post_projection arg, which indicates whether or not there should be
208
+ a post layer normalization and projector after the attention. This is
209
+ present in minicpmv2.0, but not qwen-vl.
210
+ """
211
+
212
+ def __init__(
213
+ self,
214
+ grid_size: int,
215
+ embed_dim: int,
216
+ num_heads: int,
217
+ kv_dim: Optional[int] = None,
218
+ norm_layer: Callable[[int], nn.LayerNorm] = DEFAULT_LN,
219
+ adaptive: bool = False,
220
+ do_post_projection: bool = True,
221
+ ) -> None:
222
+ super().__init__(grid_size**2,
223
+ embed_dim,
224
+ num_heads,
225
+ kv_dim,
226
+ norm_layer,
227
+ do_post_projection=do_post_projection)
228
+
229
+ self.adaptive = adaptive
230
+ pos_embed_arr = get_2d_sincos_pos_embed(embed_dim,
231
+ grid_size,
232
+ version=(2, 0))
233
+
234
+ self.pos_embed = nn.Parameter(
235
+ torch.from_numpy(pos_embed_arr).requires_grad_(False))
236
+
237
+ self.apply(self._init_weights)
238
+
239
+ def forward(
240
+ self,
241
+ x: torch.Tensor,
242
+ tgt_sizes: Optional[torch.Tensor] = None,
243
+ attn_mask: Optional[torch.Tensor] = None,
244
+ ) -> torch.Tensor:
245
+ if tgt_sizes is None:
246
+ tgt_sizes = int(math.sqrt(x.size(1)))
247
+ if self.adaptive:
248
+ pos_embed_arr = get_2d_sincos_pos_embed(self.embed_dim,
249
+ tgt_sizes,
250
+ version=(2, 0))
251
+ pos_embed = torch.from_numpy(pos_embed_arr).to(device=x.device,
252
+ dtype=x.dtype)
253
+ else:
254
+ pos_embed = get_abs_pos(self.pos_embed,
255
+ tgt_sizes).to(device=x.device,
256
+ dtype=x.dtype)
257
+
258
+ x, _ = self.kv_proj(x)
259
+ x = self.ln_kv(x).permute(1, 0, 2)
260
+
261
+ N = x.shape[1]
262
+ q = self.ln_q(self.query)
263
+ out = self.attn(
264
+ self._repeat(q, N) + self.pos_embed.unsqueeze(1),
265
+ x + pos_embed.unsqueeze(1),
266
+ x,
267
+ attn_mask=attn_mask,
268
+ )[0]
269
+ x = out.permute(1, 0, 2)
270
+ if self.do_post_projection:
271
+ x = self.ln_post(x)
272
+ x = x @ self.proj
273
+ return x
vllm/model_executor/models/__init__.py
+1
-1
@@ -51,7 +51,6 @@ _GENERATION_MODELS = {
51
"PhiForCausalLM": ("phi", "PhiForCausalLM"),
52
"Phi3ForCausalLM": ("llama", "LlamaForCausalLM"),
53
"PhiMoEForCausalLM": ("phimoe", "PhiMoEForCausalLM"),
54
- "QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
54
"Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
55
"Qwen2MoeForCausalLM": ("qwen2_moe", "Qwen2MoeForCausalLM"),
56
"RWForCausalLM": ("falcon", "FalconForCausalLM"),
@@ -88,6 +87,7 @@ _MULTIMODAL_MODELS = {
87
"PaliGemmaForConditionalGeneration"),
88
"Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
89
"UltravoxModel": ("ultravox", "UltravoxModel"),
90
+ "QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
91
}
92
_CONDITIONAL_GENERATION_MODELS = {
93
"BartModel": ("bart", "BartForConditionalGeneration"),
vllm/model_executor/models/minicpmv.py
+5
-155
@@ -26,11 +26,9 @@ import re
26
from array import array
27
from functools import partial
28
from typing import (Any, Callable, Iterable, List, Mapping, Optional, Tuple,
29
- TypedDict, Union)
29
+ TypedDict)
30
31
-import numpy as np
31
import torch
33
-import torch.nn.functional as F
32
import torch.types
33
from PIL import Image
34
from torch import nn
@@ -44,6 +42,8 @@ from vllm.logger import init_logger
42
from vllm.model_executor.layers.linear import ReplicatedLinear
43
from vllm.model_executor.layers.logits_processor import LogitsProcessor
44
from vllm.model_executor.layers.quantization import QuantizationConfig
45
+from vllm.model_executor.layers.resampler import (Resampler2,
46
+ get_2d_sincos_pos_embed)
47
from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
48
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
49
from vllm.model_executor.model_loader.utils import set_default_torch_dtype
@@ -98,101 +98,6 @@ MiniCPMVImageInputs = MiniCPMVImagePixelInputs
98
DEFAULT_LN = partial(nn.LayerNorm, eps=1e-6)
99
100
101
-def get_abs_pos(abs_pos: torch.Tensor, tgt_size: torch.Tensor):
102
- # abs_pos: L, C
103
- # tgt_size: (H, W)
104
- # return: M, C
105
- src_size = int(math.sqrt(abs_pos.size(0)))
106
- # tgt_size = int(math.sqrt(tgt_size))
107
- dtype = abs_pos.dtype
108
-
109
- return (F.interpolate(
110
- abs_pos.float().reshape(1, src_size, src_size, -1).permute(0, 3, 1, 2),
111
- size=(tgt_size[0], tgt_size[1]),
112
- mode="bicubic",
113
- align_corners=False,
114
- ).permute(0, 2, 3, 1).flatten(0, 2).to(dtype=dtype))
115
-
116
-
117
-# https://github.com/facebookresearch/mae/blob/efb2a8062c206524e35e47d04501ed4f544c0ae8/util/pos_embed.py#L20
118
-def get_2d_sincos_pos_embed(
119
- embed_dim: int,
120
- grid_size: Union[int, Tuple[int, int]],
121
- cls_token: bool = False,
122
- version: Tuple[int, int] = (2, 0),
123
-):
124
- """
125
- grid_size: int of the grid height and width
126
- return:
127
- pos_embed: [grid_size*grid_size, embed_dim] or
128
- [1+grid_size*grid_size, embed_dim] (w/ or w/o cls_token)
129
- """
130
- if isinstance(grid_size, int):
131
- grid_h_size, grid_w_size = grid_size, grid_size
132
- else:
133
- grid_h_size, grid_w_size = grid_size[0], grid_size[1]
134
-
135
- grid_h = np.arange(grid_h_size, dtype=np.float32)
136
- grid_w = np.arange(grid_w_size, dtype=np.float32)
137
- grid = np.meshgrid(grid_w, grid_h) # here w goes first
138
- grid = np.stack(grid, axis=0)
139
-
140
- if version == (2, 0):
141
- grid = grid.reshape([2, 1, grid_h_size, grid_w_size])
142
- pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid, version)
143
- if cls_token:
144
- pos_embed = np.concatenate([np.zeros([1, embed_dim]), pos_embed],
145
- axis=0)
146
- else:
147
- pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid, version)
148
- return pos_embed
149
-
150
-
151
-def get_2d_sincos_pos_embed_from_grid(embed_dim: int,
152
- grid: np.ndarray,
153
- version: Tuple[int, int] = (2, 0)):
154
- assert embed_dim % 2 == 0
155
-
156
- # use half of dimensions to encode grid_h
157
- emb_h = get_1d_sincos_pos_embed_from_grid(
158
- embed_dim // 2, grid[0], version) # (H*W, D/2) or (H, W, D/2)
159
- emb_w = get_1d_sincos_pos_embed_from_grid(
160
- embed_dim // 2, grid[1], version) # (H*W, D/2) or (H, W, D/2)
161
-
162
- if version == (2, 0):
163
- emb = np.concatenate([emb_h, emb_w], axis=1) # (H*W, D)
164
- else:
165
- emb = np.concatenate([emb_h, emb_w], axis=-1) # (H, W, D)
166
- return emb
167
-
168
-
169
-def get_1d_sincos_pos_embed_from_grid(embed_dim: int,
170
- pos: np.ndarray,
171
- version: Tuple[int, int] = (2, 0)):
172
- """
173
- embed_dim: output dimension for each position
174
- pos: a list of positions to be encoded: size (M,) / (H, W)
175
- out: (M, D) / (H, W, D)
176
- """
177
- assert embed_dim % 2 == 0
178
- omega = np.arange(embed_dim // 2, dtype=np.float32)
179
- omega /= embed_dim / 2.0
180
- omega = 1.0 / 10000**omega # (D/2,)
181
-
182
- if version == (2, 0):
183
- pos = pos.reshape(-1) # (M,)
184
- out = np.einsum("m,d->md", pos, omega) # (M, D/2), outer product
185
- emb_sin = np.sin(out) # (M, D/2)
186
- emb_cos = np.cos(out) # (M, D/2)
187
- emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D)
188
- else:
189
- out = np.einsum("hw,d->hwd", pos, omega) # (H, W, D/2), outer product
190
- emb_sin = np.sin(out) # (H, W, D/2)
191
- emb_cos = np.cos(out) # (H, W, D/2)
192
- emb = np.concatenate([emb_sin, emb_cos], axis=-1) # (H, W, D)
193
- return emb
194
-
195
-
101
class BaseResampler(nn.Module):
102
"""
103
A 2D perceiver-resampler network with one cross attention layers by
@@ -245,62 +150,6 @@ class BaseResampler(nn.Module):
150
return query.unsqueeze(1).repeat(1, N, 1)
151
152
248
-class Resampler2(BaseResampler):
249
-
250
- def __init__(
251
- self,
252
- grid_size: int,
253
- embed_dim: int,
254
- num_heads: int,
255
- kv_dim: Optional[int] = None,
256
- norm_layer: Callable[[int], nn.LayerNorm] = DEFAULT_LN,
257
- adaptive: bool = False,
258
- ) -> None:
259
- super().__init__(grid_size**2, embed_dim, num_heads, kv_dim,
260
- norm_layer)
261
-
262
- self.adaptive = adaptive
263
- pos_embed_arr = get_2d_sincos_pos_embed(embed_dim,
264
- grid_size,
265
- version=(2, 0))
266
- self.pos_embed = nn.Parameter(
267
- torch.from_numpy(pos_embed_arr).float()).requires_grad_(False)
268
-
269
- self.apply(self._init_weights)
270
-
271
- def forward(
272
- self,
273
- x: torch.Tensor,
274
- tgt_sizes: torch.Tensor,
275
- attn_mask: Optional[torch.Tensor] = None,
276
- ):
277
- if self.adaptive:
278
- pos_embed_arr = get_2d_sincos_pos_embed(self.embed_dim,
279
- tgt_sizes,
280
- version=(2, 0))
281
- pos_embed = torch.from_numpy(pos_embed_arr).to(device=x.device,
282
- dtype=x.dtype)
283
- else:
284
- pos_embed = get_abs_pos(self.pos_embed, tgt_sizes)
285
-
286
- x, _ = self.kv_proj(x)
287
- x = self.ln_kv(x).permute(1, 0, 2)
288
-
289
- N = x.shape[1]
290
- q = self.ln_q(self.query)
291
- out = self.attn(
292
- self._repeat(q, N) + self.pos_embed.unsqueeze(1),
293
- x + pos_embed.unsqueeze(1),
294
- x,
295
- attn_mask=attn_mask,
296
- )[0]
297
- x = out.permute(1, 0, 2)
298
-
299
- x = self.ln_post(x)
300
- x = x @ self.proj
301
- return x
302
-
303
-
153
class Resampler2_5(BaseResampler):
154
155
def __init__(
@@ -782,7 +631,8 @@ class MiniCPMV2_0(MiniCPMVBaseModel):
631
num_heads=embed_dim // 128,
632
grid_size=int(math.sqrt(self.config.query_num)),
633
kv_dim=vision_dim,
785
- adaptive=True,
634
+ adaptive=False,
635
+ do_post_projection=True,
636
)
637
638
return resampler
vllm/model_executor/models/qwen.py
+667
-27
@@ -4,36 +4,402 @@
4
# Copyright (c) Alibaba Cloud.
5
# LICENSE: https://huggingface.co/Qwen/Qwen-7B/blob/main/LICENSE
6
"""Inference-only QWen model compatible with HuggingFace weights."""
7
-from typing import Any, Dict, Iterable, List, Optional, Tuple
7
8
+import math
9
+import re
10
+from array import array
11
+from functools import partial
12
+from typing import (Any, Callable, Dict, Iterable, List, Literal, Mapping,
13
+ Optional, Tuple, TypedDict, Union)
14
+
15
+import numpy as np
16
import torch
17
+from PIL import Image
18
from torch import nn
19
+from torchvision import transforms
20
+from torchvision.transforms import InterpolationMode
21
from transformers import PretrainedConfig
22
23
from vllm.attention import Attention, AttentionMetadata
14
-from vllm.config import CacheConfig
24
+from vllm.config import CacheConfig, MultiModalConfig
25
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
16
-from vllm.model_executor.layers.activation import SiluAndMul
26
+from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
27
+from vllm.logger import init_logger
28
+from vllm.model_executor.layers.activation import SiluAndMul, get_act_fn
29
from vllm.model_executor.layers.layernorm import RMSNorm
18
-from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
30
+from vllm.model_executor.layers.linear import (ColumnParallelLinear,
31
+ MergedColumnParallelLinear,
32
QKVParallelLinear,
33
RowParallelLinear)
34
from vllm.model_executor.layers.logits_processor import LogitsProcessor
35
from vllm.model_executor.layers.quantization.base_config import (
36
QuantizationConfig)
37
+from vllm.model_executor.layers.resampler import Resampler2, get_abs_pos
38
from vllm.model_executor.layers.rotary_embedding import get_rope
39
from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
40
from vllm.model_executor.layers.vocab_parallel_embedding import (
41
ParallelLMHead, VocabParallelEmbedding)
42
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
43
+from vllm.model_executor.models.interfaces import SupportsMultiModal
44
from vllm.model_executor.sampling_metadata import SamplingMetadata
30
-from vllm.sequence import IntermediateTensors
31
-from vllm.utils import print_warning_once
45
+from vllm.multimodal import MULTIMODAL_REGISTRY
46
+from vllm.multimodal.base import MultiModalInputs
47
+from vllm.multimodal.utils import cached_get_tokenizer
48
+from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
49
+ SequenceData)
50
+
51
+from .utils import flatten_bn, is_pp_missing_parameter, make_layers
52
+
53
+logger = init_logger(__name__)
54
+
55
+# NOTE: Qwen models have a few other special tags, e.g., ref, bbox, quad;
56
+# for the time being, these tags are not considered as special at encoding
57
+# time. This may change as VLLMs multimodal API changes in the future.
58
+IMG_START = "<img>"
59
+IMG_END = "</img>"
60
+IMG_PAD = "<imgpad>"
61
+# Image context is fixed at 256 for all images
62
+MAX_QWEN_IMG_TOKENS = 256
63
+# Image normalization params
64
+CLIP_MEAN = (0.48145466, 0.4578275, 0.40821073)
65
+CLIP_STD = (0.26862954, 0.26130258, 0.27577711)
66
+
67
+
68
+class QwenImagePixelInputs(TypedDict):
69
+ type: Literal["pixel_values"]
70
+ data: torch.Tensor
71
+ """
72
+ Shape: `(batch_size * num_images, 3, image_size, image_size)`
73
+
74
+ Note that image_size is the value in the vision config to which we resize
75
+ the image to in the normalization transform. Currently multi-image support
76
+ can only be leveraged by passing image embeddings directly.
77
+ """
78
+
79
+
80
+class QwenImageEmbeddingInputs(TypedDict):
81
+ type: Literal["image_embeds"]
82
+ data: torch.Tensor
83
+ """Shape: `(batch_size * num_images, 256, hidden_size)`
84
+
85
+ `hidden_size` must match the hidden size of the language model backbone
86
+ and is stored in the visual config of the model if we have one.
87
+ """
88
+
89
+
90
+QwenImageInputs = Union[QwenImagePixelInputs, QwenImageEmbeddingInputs]
91
+
92
+
93
+class VisualAttention(nn.Module):
94
+ """self-attention layer class.
95
+ Self-attention layer takes input with size [s, b, h]
96
+ and returns output of the same size.
97
+ """
98
+
99
+ def __init__(
100
+ self,
101
+ embed_dim: int,
102
+ num_heads: int,
103
+ bias: bool = True,
104
+ kdim: Optional[int] = None,
105
+ vdim: Optional[int] = None,
106
+ ):
107
+ super().__init__()
108
+ self.embed_dim = embed_dim
109
+ self.kdim = kdim if kdim is not None else embed_dim
110
+ self.vdim = vdim if vdim is not None else embed_dim
111
+ self._qkv_same_embed_dim = self.kdim == embed_dim \
112
+ and self.vdim == embed_dim
113
+
114
+ self.num_heads = num_heads
115
+
116
+ # Per attention head and per partition values.
117
+ assert embed_dim % num_heads == 0
118
+ self.hidden_size_per_attention_head = embed_dim // num_heads
119
+ self.num_attention_heads_per_partition = num_heads
120
+ self.hidden_size_per_partition = embed_dim
121
+
122
+ # Strided linear layer.
123
+ assert self._qkv_same_embed_dim, \
124
+ 'Visual Attention implementation only supports self-attention'
125
+ self.in_proj = nn.Linear(embed_dim, 3 * embed_dim)
126
+ self.out_proj = nn.Linear(embed_dim, embed_dim)
127
+ self.norm_factor = math.sqrt(self.hidden_size_per_attention_head)
128
+
129
+ def forward(
130
+ self,
131
+ x: torch.Tensor,
132
+ attn_mask: Optional[torch.Tensor] = None,
133
+ ) -> torch.Tensor:
134
+ # query/key/value: [sq, b, h]
135
+ sq, b, _ = x.size()
136
+ mixed_x_layer = self.in_proj(x)
137
+
138
+ # [sq, b, (np * 3 * hn)] --> [sq, b, np, 3 * hn]
139
+ new_tensor_shape = mixed_x_layer.size()[:-1] + \
140
+ (self.num_attention_heads_per_partition,
141
+ 3 * self.hidden_size_per_attention_head)
142
+ mixed_x_layer = mixed_x_layer.view(*new_tensor_shape)
143
+
144
+ # [sq, b, np, 3 * hn] --> 3 [sq, b, np, hn]
145
+ query_layer, key_layer, value_layer = mixed_x_layer.split(
146
+ self.hidden_size_per_attention_head, dim=-1)
147
+
148
+ # [sq, b, np, hn] -> [sq, b * np, hn]
149
+ query_layer = query_layer.view(
150
+ sq, b * self.num_attention_heads_per_partition,
151
+ self.hidden_size_per_attention_head).transpose(0, 1)
152
+ # [sk, b, np, hn] -> [sk, b * np, hn]
153
+ key_layer = key_layer.view(
154
+ sq, b * self.num_attention_heads_per_partition,
155
+ self.hidden_size_per_attention_head).transpose(0, 1)
156
+
157
+ q_scaled = query_layer / self.norm_factor
158
+ if attn_mask is not None:
159
+ attention_probs = torch.baddbmm(attn_mask, q_scaled,
160
+ key_layer.transpose(-2, -1))
161
+ else:
162
+ attention_probs = torch.bmm(q_scaled, key_layer.transpose(-2, -1))
163
+ attention_probs = attention_probs.softmax(dim=-1)
164
+
165
+ value_layer = value_layer.view(
166
+ sq, b * self.num_attention_heads_per_partition,
167
+ self.hidden_size_per_attention_head).transpose(0, 1)
168
+
169
+ # matmul: [b * np, sq, hn]
170
+ context_layer = torch.bmm(attention_probs, value_layer)
171
33
-from .utils import is_pp_missing_parameter, make_layers
172
+ # change view [b, np, sq, hn]
173
+ context_layer = context_layer.view(
174
+ b, self.num_attention_heads_per_partition, sq,
175
+ self.hidden_size_per_attention_head)
176
+
177
+ # [b, np, sq, hn] --> [sq, b, np, hn]
178
+ context_layer = context_layer.permute(2, 0, 1, 3).contiguous()
179
+
180
+ # [sq, b, np, hn] --> [sq, b, hp]
181
+ new_context_layer_shape = context_layer.size()[:-2] + \
182
+ (self.hidden_size_per_partition,)
183
+ context_layer = context_layer.view(*new_context_layer_shape)
184
+
185
+ output = self.out_proj(context_layer)
186
+
187
+ return output
188
+
189
+
190
+class QwenVMLP(nn.Module):
191
+ """MLP for the visual component of the Qwen model."""
192
+
193
+ def __init__(
194
+ self,
195
+ hidden_size: int,
196
+ intermediate_size: int,
197
+ quant_config: Optional[QuantizationConfig] = None,
198
+ ):
199
+ super().__init__()
200
+ self.c_fc = ColumnParallelLinear(hidden_size,
201
+ intermediate_size,
202
+ bias=True,
203
+ quant_config=quant_config)
204
+ self.act_fn = get_act_fn("gelu", quant_config, intermediate_size)
205
+ self.c_proj = RowParallelLinear(
206
+ intermediate_size,
207
+ hidden_size,
208
+ bias=True,
209
+ quant_config=quant_config,
210
+ )
211
+
212
+ def forward(self, x):
213
+ x, _ = self.c_fc(x)
214
+ x = self.act_fn(x)
215
+ x, _ = self.c_proj(x)
216
+ return x
217
+
218
+
219
+class VisualAttentionBlock(nn.Module):
220
+
221
+ def __init__(
222
+ self,
223
+ d_model: int,
224
+ n_head: int,
225
+ mlp_ratio: float = 4.0,
226
+ norm_layer: Callable = nn.LayerNorm,
227
+ quant_config: Optional[QuantizationConfig] = None,
228
+ ):
229
+ super().__init__()
230
+
231
+ self.ln_1 = norm_layer(d_model)
232
+ self.ln_2 = norm_layer(d_model)
233
+ mlp_width = int(d_model * mlp_ratio)
234
+ self.attn = VisualAttention(d_model, n_head)
235
+ self.mlp = QwenVMLP(
236
+ hidden_size=d_model,
237
+ intermediate_size=mlp_width,
238
+ quant_config=quant_config,
239
+ )
240
+
241
+ def attention(
242
+ self,
243
+ x: torch.Tensor,
244
+ attn_mask: Optional[torch.Tensor] = None,
245
+ ) -> torch.Tensor:
246
+ attn_mask = attn_mask.to(x.dtype) if attn_mask is not None else None
247
+ return self.attn(x, attn_mask=attn_mask)
248
+
249
+ def forward(
250
+ self,
251
+ x: torch.Tensor,
252
+ attn_mask: Optional[torch.Tensor] = None,
253
+ ) -> torch.Tensor:
254
+ x = x + self.attention(self.ln_1(x), attn_mask=attn_mask)
255
+ x = x + self.mlp(self.ln_2(x))
256
+ return x
257
+
258
+
259
+class TransformerBlock(nn.Module):
260
+
261
+ def __init__(
262
+ self,
263
+ width: int,
264
+ layers: int,
265
+ heads: int,
266
+ mlp_ratio: float = 4.0,
267
+ norm_layer: Callable = nn.LayerNorm,
268
+ quant_config: Optional[QuantizationConfig] = None,
269
+ ):
270
+ super().__init__()
271
+ self.width = width
272
+ self.layers = layers
273
+
274
+ self.resblocks = nn.ModuleList([
275
+ VisualAttentionBlock(width,
276
+ heads,
277
+ mlp_ratio,
278
+ norm_layer=norm_layer,
279
+ quant_config=quant_config)
280
+ for _ in range(layers)
281
+ ])
282
+
283
+ def get_cast_dtype(self) -> torch.dtype:
284
+ return self.resblocks[0].mlp.c_fc.weight.dtype
285
+
286
+ def get_cast_device(self) -> torch.device:
287
+ return self.resblocks[0].mlp.c_fc.weight.device
288
+
289
+ def forward(self,
290
+ x: torch.Tensor,
291
+ attn_mask: Optional[torch.Tensor] = None) -> torch.Tensor:
292
+ for r in self.resblocks:
293
+ x = r(x, attn_mask=attn_mask)
294
+ return x
295
+
296
+
297
+class VisionTransformer(nn.Module):
298
+
299
+ def __init__(self,
300
+ image_size: int,
301
+ patch_size: int,
302
+ width: int,
303
+ layers: int,
304
+ heads: int,
305
+ mlp_ratio: float,
306
+ n_queries: int = 256,
307
+ output_dim: int = 512,
308
+ image_start_id: int = 151857,
309
+ quant_config: Optional[QuantizationConfig] = None,
310
+ **kwargs):
311
+ super().__init__()
312
+ image_height, image_width = self.image_size = (image_size, image_size)
313
+ patch_height, patch_width = self.patch_size = (patch_size, patch_size)
314
+ self.grid_size = (image_height // patch_height,
315
+ image_width // patch_width)
316
+ self.output_dim = output_dim
317
+ self.conv1 = nn.Conv2d(in_channels=3,
318
+ out_channels=width,
319
+ kernel_size=patch_size,
320
+ stride=patch_size,
321
+ bias=False)
322
+
323
+ # class embeddings and positional embeddings
324
+ scale = width**-0.5
325
+ self.positional_embedding = nn.Parameter(scale *
326
+ torch.randn(256, width))
327
+
328
+ norm_layer = partial(nn.LayerNorm, eps=1e-6)
329
+
330
+ self.ln_pre = norm_layer(width)
331
+ self.transformer = TransformerBlock(width,
332
+ layers,
333
+ heads,
334
+ mlp_ratio,
335
+ norm_layer=norm_layer,
336
+ quant_config=quant_config)
337
+
338
+ self.attn_pool = Resampler2(
339
+ grid_size=int(math.sqrt(n_queries)),
340
+ embed_dim=output_dim,
341
+ num_heads=output_dim // 128,
342
+ kv_dim=width,
343
+ norm_layer=norm_layer,
344
+ adaptive=False,
345
+ do_post_projection=False,
346
+ ).to(
347
+ device=self.positional_embedding.device,
348
+ dtype=self.positional_embedding.dtype,
349
+ )
350
+
351
+ self.ln_post = norm_layer(output_dim)
352
+ self.proj = nn.Parameter(
353
+ (output_dim**-0.5) * torch.randn(output_dim, output_dim))
354
+ self.image_start_id = image_start_id
355
+ self.image_end_id = image_start_id + 1
356
+
357
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
358
+ x = x.to(
359
+ dtype=self.transformer.get_cast_dtype(),
360
+ device=self.transformer.get_cast_device(),
361
+ )
362
+
363
+ # to patches
364
+ x = self.conv1(x) # shape = [*, width, grid, grid]
365
+ x = x.reshape(x.shape[0], x.shape[1],
366
+ -1) # shape = [*, width, grid ** 2]
367
+ x = x.permute(0, 2, 1) # shape = [*, grid ** 2, width]
368
+
369
+ x = x + get_abs_pos(self.positional_embedding, int(math.sqrt(
370
+ x.size(1))))
371
+
372
+ x = self.ln_pre(x)
373
+
374
+ x = x.permute(1, 0, 2) # NLD -> LND
375
+ x = self.transformer(x)
376
+ x = x.permute(1, 0, 2) # LND -> NLD
377
+
378
+ x = self.attn_pool(x)
379
+ x = self.ln_post(x)
380
+ x = x @ self.proj
381
+
382
+ return x
383
+
384
+ def get_image_positions(self,
385
+ input_ids: torch.Tensor) -> Optional[torch.Tensor]:
386
+ """Given the input IDs, extracts start/stop points corresponding to
387
+ images.
388
+
389
+ args:
390
+ Returns:
391
+ Optional torch tensor corresponding to start/stop pairs of images.
392
+ """
393
+ if torch.any(input_ids == self.image_start_id):
394
+ bos_pos = torch.where(input_ids == self.image_start_id)
395
+ eos_pos = torch.where(input_ids == self.image_end_id)
396
+ return torch.stack((bos_pos[0], eos_pos[0]), dim=1)
397
+ return None
398
399
400
class QWenMLP(nn.Module):
401
+ """MLP for the language component of the Qwen model, which contains a
402
+ MergedColumnParallelLinear merging 2 outputs via silu activation."""
403
404
def __init__(
405
self,
@@ -56,7 +422,7 @@ class QWenMLP(nn.Module):
422
"Only silu is supported for now.")
423
self.act_fn = SiluAndMul()
424
59
- def forward(self, x):
425
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
426
gate_up, _ = self.gate_up_proj(x)
427
x = self.act_fn(gate_up)
428
x, _ = self.c_proj(x)
@@ -203,6 +569,9 @@ class QWenModel(nn.Module):
569
lambda prefix: QWenBlock(config, cache_config, quant_config),
570
prefix=f"{prefix}.h")
571
self.ln_f = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
572
+ self.visual = VisionTransformer(**config.visual,
573
+ quant_config=quant_config) if hasattr(
574
+ config, "visual") else None
575
576
def forward(
577
self,
@@ -211,9 +580,33 @@ class QWenModel(nn.Module):
580
kv_caches: List[torch.Tensor],
581
attn_metadata: AttentionMetadata,
582
intermediate_tensors: Optional[IntermediateTensors],
583
+ pixel_values: Optional[QwenImageInputs],
584
) -> torch.Tensor:
585
+ img_pos = None
586
+ # If pixel / visual embeddings are provided, this is a visual model
587
+ if pixel_values is not None and self.visual is not None:
588
+ if pixel_values["type"] != "image_embeds":
589
+ image_embeds = self.visual(pixel_values["data"])
590
+ else:
591
+ image_embeds = pixel_values["data"]
592
+
593
+ # features should be of shape (# images, 256, hidden_dim)
594
+ img_pos = self.visual.get_image_positions(input_ids)
595
+ if isinstance(
596
+ img_pos,
597
+ np.ndarray) and img_pos.shape[0] != image_embeds.shape[0]:
598
+ raise ValueError(
599
+ f"Number of placeholders: {img_pos.shape[0]} "
600
+ f"does not match number of images {image_embeds.shape[0]}."
601
+ )
602
+
603
if get_pp_group().is_first_rank:
604
hidden_states = self.wte(input_ids)
605
+ # Merge the image embeddings into the hidden states if actually have
606
+ # visual features and the corresponding image tokens
607
+ if img_pos is not None:
608
+ for idx, (img_bos, img_eos) in enumerate(img_pos):
609
+ hidden_states[img_bos + 1:img_eos] = image_embeds[idx]
610
residual = None
611
else:
612
assert intermediate_tensors is not None
@@ -237,16 +630,241 @@ class QWenModel(nn.Module):
630
return hidden_states
631
632
240
-class QWenLMHeadModel(nn.Module):
633
+def get_image_text(image_num: int, padding: bool) -> str:
634
+ """Retrieves a placeholder text that when tokenized, will be expanded with
635
+ image pads.
636
+
637
+ Args:
638
+ image_num: The number of the image that we want a text prompt for.
639
+ Images should be indexed starting at 1.
640
+ padding: Whether or not padding should be manually added.
641
+
642
+ Returns:
643
+ Text placeholder prompt for the image being considered.
644
+ """
645
+ image_start = f"Picture {image_num}: {IMG_START}"
646
+ image_end = f"{IMG_END}\n"
647
+ if not padding:
648
+ return f"{image_start}{image_end}"
649
+ return f"{image_start}{MAX_QWEN_IMG_TOKENS * IMG_PAD}{image_end}"
650
+
651
+
652
+def input_processor_for_qwen(ctx: InputContext,
653
+ llm_inputs: LLMInputs) -> LLMInputs:
654
+ """Processes the inputs, which may or may not be multimodal.
655
+ Multimodal inputs will only be processed if the model has a "visual"
656
+ component in its model config, otherwise they'll be ignored.
657
+
658
+ Args:
659
+ ctx: Context of the loaded model.
660
+ llm_inputs: LLM inputs which may have a multi_modal_data attribute.
661
+
662
+ Returns:
663
+ If the model is language only or not multimodal inputs were provided,
664
+ returns llm_inputs unmodified. Otherwise, processes the multimodal
665
+ images / image embeddings and adds the fixed-length image placeholders.
666
+ """
667
+ multi_modal_data = llm_inputs.get("multi_modal_data")
668
+
669
+ # Only process images if we have multimodal data and a visual config
670
+ hf_config = ctx.get_hf_config()
671
+ if (multi_modal_data is None or "image" not in multi_modal_data
672
+ or not hasattr(hf_config, "visual")):
673
+ return llm_inputs
674
+
675
+ prompt = llm_inputs.get("prompt")
676
+ prompt_token_ids = llm_inputs["prompt_token_ids"]
677
+ model_config = ctx.model_config
678
+ tokenizer = cached_get_tokenizer(model_config.tokenizer,
679
+ trust_remote_code=True)
680
+ image_data = multi_modal_data["image"]
681
+ if isinstance(image_data, torch.Tensor):
682
+ num_dims = len(image_data.shape)
683
+ if num_dims < 2 or num_dims > 3:
684
+ raise ValueError(
685
+ f"Expected img embeds to be have 3 dimensions, got {num_dims}")
686
+ num_images = 1 if num_dims == 2 else image_data.shape[0]
687
+ else:
688
+ # TODO - handle multiple image inputs once the API is solidified
689
+ num_images = 1
690
+
691
+ if prompt is None:
692
+ prompt = tokenizer.decode(prompt_token_ids)
693
+
694
+ # Drops anything between <img>/</img> tags; encoding with the tokenizer
695
+ # will automatically add the image pads for the context.
696
+ new_prompt, num_matched_images = re.subn(
697
+ r"(Picture \d*: <img>).*?(<\/img>\n)",
698
+ r"\1\2",
699
+ prompt,
700
+ )
701
+
702
+ if num_matched_images != num_images:
703
+ logger.warning(
704
+ "Number of matched image placeholders %s doesn't match the number "
705
+ "of expected images %s; check your placeholder formatting.",
706
+ num_matched_images, num_images)
707
+
708
+ new_prompt_token_ids = tokenizer.encode(new_prompt)
709
+
710
+ return LLMInputs(prompt=new_prompt,
711
+ prompt_token_ids=new_prompt_token_ids,
712
+ multi_modal_data=multi_modal_data)
713
+
714
+
715
+def input_mapper_for_qwen(ctx: InputContext, data: object) -> MultiModalInputs:
716
+ """Maps the input data to its MultiModalInputs (if any).
717
+
718
+ Args:
719
+ ctx: Context of the loaded model.
720
+ data: data potentially containing image/image embeddings to be mapped
721
+ to pixel_values in .forward() for a visual QWenLMHeadModel model.
722
+
723
+ Returns:
724
+ MultiModalInputs containing the stacked normalized images tensor or
725
+ image embeddings.
726
+ """
727
+ # Early exit if we have provided an image to a language only Qwen model
728
+ hf_config = ctx.get_hf_config()
729
+ if not hasattr(hf_config, "visual"):
730
+ logger.warning(
731
+ "Images were provided but this model has no visual config; "
732
+ "multimodal inputs will not be forwarded to the model.")
733
+ return MultiModalInputs()
734
+
735
+ model_config = ctx.model_config
736
+ tokenizer = cached_get_tokenizer(model_config.tokenizer,
737
+ trust_remote_code=True)
738
+
739
+ image_pair_tok = tokenizer.encode(IMG_START + IMG_END,
740
+ add_special_tokens=False,
741
+ return_tensors="pt").squeeze()
742
+ image_start_id = image_pair_tok[0]
743
+ image_end_id = image_pair_tok[-1]
744
+ if (image_start_id + 1) != image_end_id:
745
+ raise ValueError(
746
+ f"Found image end ID {image_end_id}, but expected {IMG_START} + 1")
747
+ if len(image_pair_tok) != (MAX_QWEN_IMG_TOKENS + 2):
748
+ raise ValueError(
749
+ f"Expected image context length of {MAX_QWEN_IMG_TOKENS}, "
750
+ f"but got {image_pair_tok - 2}")
751
+
752
+ hf_config = ctx.get_hf_config()
753
+ image_size = hf_config.visual["image_size"]
754
+ img_emb_size = hf_config.visual["output_dim"]
755
+
756
+ if isinstance(data, torch.Tensor):
757
+ # It's expected that our values have already been processed
758
+ # by the visual transformer; shape is expected to be:
759
+ # (# images, 256, hidden_size)
760
+ if len(data.shape) == 2:
761
+ # Assume only one image embed was provided; unsqueeze the extra dim
762
+ data = data.unsqueeze(0)
763
+ if len(data.shape) != 3 or data.shape[
764
+ 1] != MAX_QWEN_IMG_TOKENS or data.shape[2] != img_emb_size:
765
+ raise ValueError(
766
+ "Expected image embeds to be a tensor of shape"
767
+ f"[# images, {MAX_QWEN_IMG_TOKENS}, {img_emb_size}], but "
768
+ f"received shape [{data.shape}]")
769
+ pixel_values = data
770
+
771
+ else:
772
+ transform = build_normalization_transform(image_size)
773
+ # TODO - handle multiple image inputs once the API is solidified
774
+ transformed_images = [transform(data)]
775
+ pixel_values = torch.stack(transformed_images, dim=0)
776
+ return MultiModalInputs({"pixel_values": pixel_values})
777
+
778
+
779
+def build_normalization_transform(image_size: int) -> transforms.Compose:
780
+ """Builds a normalization transform which can be applied to one or
781
+ more input images from which we want to extract visual features.
782
+
783
+ Args:
784
+ image_size: size of the image to be processed for visual embeddings.
785
+
786
+ Returns:
787
+ Callable transform for normalizing and resizing one RGB image.
788
+ """
789
+ return transforms.Compose([
790
+ transforms.Resize((image_size, image_size),
791
+ interpolation=InterpolationMode.BICUBIC),
792
+ transforms.ToTensor(),
793
+ transforms.Normalize(mean=CLIP_MEAN, std=CLIP_STD),
794
+ ])
795
+
796
+
797
+def dummy_data_for_qwen(
798
+ ctx: InputContext,
799
+ seq_len: int,
800
+ mm_counts: Mapping[str, int],
801
+) -> Tuple[SequenceData, Optional[Dict]]:
802
+ """Build dummy data for warming up Qwen models; this will only contain text
803
+ matching the defaults for VLLM unless the model has a visual config.
804
+
805
+ Args:
806
+ ctx: Context of the loaded model.
807
+ seq_len: Number of tokens in the text sequence.
808
+ mm_counts: multimodal data counts.
809
+
810
+ Returns:
811
+ Tuple containing sequential and multimodal data.
812
+ """
813
+ hf_config = ctx.get_hf_config()
814
+
815
+ # The presence of a visual config indicates this is a multimodal model.
816
+ # If we don't have it, the model is considered an LLM for warmup purposes.
817
+ if not hasattr(hf_config, "visual"):
818
+ seq_data = SequenceData(array(VLLM_TOKEN_ID_ARRAY_TYPE, [0] * seq_len))
819
+ mm_data = None
820
+ return seq_data, mm_data
821
+
822
+ # We have a visual component - use images to warm up
823
+ num_images = mm_counts["image"]
824
+ model_config = ctx.model_config
825
+ tokenizer = cached_get_tokenizer(model_config.tokenizer,
826
+ trust_remote_code=True)
827
+
828
+ # Build the image prompts with no imgpads; the tokenizer will add img pads
829
+ image_prompt = ''.join(
830
+ [get_image_text(idx, False) for idx in range(1, num_images + 1)])
831
+ toks = tokenizer.encode(image_prompt, add_special_tokens=False)
832
+
833
+ # Make sure we actually get the fixed context size per tok padding
834
+ num_pads = toks.count(tokenizer.encode(IMG_PAD)[0])
835
+ if num_pads != (num_images * MAX_QWEN_IMG_TOKENS):
836
+ raise ValueError(
837
+ f"Tokenized dummy data should encode {MAX_QWEN_IMG_TOKENS} pads"
838
+ f" per image, but got {num_pads} pads for {num_images} image(s)"
839
+ " in total. Are you using a qwen tokenizer?")
840
+
841
+ # Ensure the number of tokens is at minimum the sequence length provided
842
+ if len(toks) < seq_len:
843
+ toks += [0] * (seq_len - len(toks))
844
+
845
+ # Build the input images; width/height doesn't actually matter here since
846
+ # the data will get resized and the # of tokens per image is constant
847
+ image = Image.new("RGB", (224, 224), color=0)
848
+ mm_data = {"image": image if num_images == 1 else [image] * num_images}
849
+ return SequenceData(array(VLLM_TOKEN_ID_ARRAY_TYPE, toks)), mm_data
850
+
851
+
852
+@MULTIMODAL_REGISTRY.register_image_input_mapper(input_mapper_for_qwen)
853
+@MULTIMODAL_REGISTRY.register_max_image_tokens(MAX_QWEN_IMG_TOKENS)
854
+@INPUT_REGISTRY.register_dummy_data(dummy_data_for_qwen)
855
+@INPUT_REGISTRY.register_input_processor(input_processor_for_qwen)
856
+class QWenLMHeadModel(nn.Module, SupportsMultiModal):
857
858
def __init__(
859
self,
860
config: PretrainedConfig,
861
+ multimodal_config: MultiModalConfig,
862
cache_config: Optional[CacheConfig] = None,
863
quant_config: Optional[QuantizationConfig] = None,
864
):
865
super().__init__()
866
self.config = config
867
+ self.multimodal_config = multimodal_config
868
self.quant_config = quant_config
869
self.transformer = QWenModel(config, cache_config, quant_config)
870
self.lm_head = ParallelLMHead(config.vocab_size,
@@ -257,16 +875,47 @@ class QWenLMHeadModel(nn.Module):
875
self.logits_processor = LogitsProcessor(config.vocab_size)
876
self.sampler = Sampler()
877
260
- def forward(
261
- self,
262
- input_ids: torch.Tensor,
263
- positions: torch.Tensor,
264
- kv_caches: List[torch.Tensor],
265
- attn_metadata: AttentionMetadata,
266
- intermediate_tensors: Optional[IntermediateTensors] = None,
267
- ) -> torch.Tensor:
878
+ def _get_image_input_type(
879
+ self,
880
+ pixel_values: Optional[torch.Tensor]) -> Optional[QwenImageInputs]:
881
+ """Determines if the provided pixel_values are normalized pixel values
882
+ or image embeddings.
883
+
884
+ Args:
885
+ pixel_values: Optional data to processed into visual embeddings.
886
+
887
+ Returns:
888
+ None of the QwenImageInputs type used to determine whether or not
889
+ the visual transformer needs to process the pixel_values.
890
+ """
891
+ if pixel_values is not None and self.transformer.visual is not None:
892
+ pixel_values = flatten_bn(pixel_values)
893
+ if len(pixel_values.shape) == 3 and pixel_values.shape[
894
+ 1] == MAX_QWEN_IMG_TOKENS and pixel_values.shape[
895
+ 2] == self.config.visual["output_dim"]:
896
+ return QwenImageEmbeddingInputs(
897
+ type="image_embeds",
898
+ data=pixel_values,
899
+ )
900
+ else:
901
+ # If we have the wrong shape, assume we still need to process
902
+ return QwenImagePixelInputs(
903
+ type="pixel_values",
904
+ data=pixel_values,
905
+ )
906
+ return None
907
+
908
+ def forward(self,
909
+ input_ids: torch.Tensor,
910
+ positions: torch.Tensor,
911
+ kv_caches: List[torch.Tensor],
912
+ attn_metadata: AttentionMetadata,
913
+ intermediate_tensors: Optional[IntermediateTensors] = None,
914
+ pixel_values: Optional[torch.Tensor] = None) -> torch.Tensor:
915
+ pixel_values = self._get_image_input_type(pixel_values)
916
hidden_states = self.transformer(input_ids, positions, kv_caches,
269
- attn_metadata, intermediate_tensors)
917
+ attn_metadata, intermediate_tensors,
918
+ pixel_values)
919
return hidden_states
920
921
def make_empty_intermediate_tensors(
@@ -328,15 +977,6 @@ class QWenLMHeadModel(nn.Module):
977
# Skip loading extra bias for GPTQ models.
978
if name.endswith(".bias") and name not in params_dict:
979
continue
331
- # Skip loading visual weights to support Qwen-VL models
332
- # in cases with text-only inputs
333
- # TODO: add support for Qwen-VL
334
- if (name not in params_dict
335
- and name.startswith("transformer.visual.")):
336
- print_warning_once(
337
- "Only text inputs are allowed. Images won't be handled "
338
- "until Qwen-VL models are fully supported.")
339
- continue
980
# Skip layers on other devices.
981
if is_pp_missing_parameter(name, self):
982
continue