@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9da25a88aa
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[MODEL] Qwen Multimodal Support (Qwen-VL / Qwen-VL-Chat) (#8029)

Signed-off-by: Alex-Brooks <Alex.Brooks@ibm.com> Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>

Alex Brooks committed Sep 5, 2024 at 06:48 UTC 9da25a88aa35da4b5ad7da545e6189e08c5f52f4
8 files changed +1110 -208
docs/source/models/supported_models.rst
+5
@@ -242,6 +242,11 @@ Multimodal Language Models
242 - Image\ :sup:`+`
243 - :code:`openbmb/MiniCPM-V-2` (see note), :code:`openbmb/MiniCPM-Llama3-V-2_5`, :code:`openbmb/MiniCPM-V-2_6`, etc.
244 -
245 + * - :code:`QWenLMHeadModel`
246 + - Qwen
247 + - Image
248 + - :code:`Qwen/Qwen-VL`, :code:`Qwen/Qwen-VL-Chat`, etc.
249 + -
250 * - :code:`UltravoxModel`
251 - Ultravox
252 - Audio\ :sup:`E+`
examples/offline_inference_vision_language.py
+15
@@ -159,6 +159,20 @@ def run_blip2(question):
159 return llm, prompt, stop_token_ids
160
161
162 +# Qwen
163 +def run_qwen_vl(question):
164 +
165 + llm = LLM(
166 + model="Qwen/Qwen-VL",
167 + trust_remote_code=True,
168 + max_num_seqs=5,
169 + )
170 +
171 + prompt = f"{question}Picture 1: <img></img>\n"
172 + stop_token_ids = None
173 + return llm, prompt, stop_token_ids
174 +
175 +
176 model_example_map = {
177 "llava": run_llava,
178 "llava-next": run_llava_next,
@@ -169,6 +183,7 @@ model_example_map = {
183 "minicpmv": run_minicpmv,
184 "blip-2": run_blip2,
185 "internvl_chat": run_internvl,
186 + "qwen_vl": run_qwen_vl,
187 }
188
189
tests/models/test_qwen.py
+142 -25
@@ -1,48 +1,165 @@
1 -from typing import Type
1 +import pathlib
2 +from typing import List, Optional, Type
3
4 import pytest
5
5 -from ..conftest import HfRunner, VllmRunner
6 +from vllm.multimodal.utils import rescale_image_size
7 +
8 +from ..conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
9 from .utils import check_logprobs_close
10
8 -models = ["qwen/qwen-vl"]
11 +pytestmark = pytest.mark.vlm
12
13 +text_only_models = [
14 + "Qwen/Qwen-7B-Chat" # Has no visual component
15 +]
16
11 -@pytest.mark.parametrize("dtype", ["half"])
12 -@pytest.mark.parametrize("max_tokens", [32])
13 -@pytest.mark.parametrize("num_logprobs", [5])
14 -@pytest.mark.parametrize("model", models)
15 -def test_text_only_qwen_model(
17 +multimodal_models = ["Qwen/Qwen-VL"]
18 +
19 +HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
20 + "stop_sign":
21 + "Picture 1: <img></img>\nWhat's the content of the image?: ",
22 + "cherry_blossom":
23 + "Picture 1: <img></img>\nWhat is the season?: ",
24 +})
25 +
26 +
27 +### Tests for multimodal Qwen models
28 +def run_test(
29 + tmp_path: pathlib.PosixPath,
30 hf_runner: Type[HfRunner],
31 vllm_runner: Type[VllmRunner],
18 - example_prompts,
32 + image_assets: _ImageAssets,
33 model: str,
34 *,
35 + size_factors: List[float],
36 dtype: str,
37 max_tokens: int,
38 num_logprobs: int,
39 + tensor_parallel_size: int,
40 + distributed_executor_backend: Optional[str] = None,
41 ):
25 - # This test checks language inputs only, since the visual component
26 - # for qwen-vl is still unsupported in VLLM. In the near-future, the
27 - # implementation and this test will be extended to consider
28 - # visual inputs as well.
42 + """Inference result should be the same between hf and vllm.
43 +
44 + All the image fixtures for the test is under tests/images.
45 + For huggingface runner, we provide the PIL images as input.
46 + For vllm runner, we provide MultiModalDataDict objects
47 + and corresponding MultiModalConfig as input.
48 + Note, the text input is also adjusted to abide by vllm contract.
49 + The text output is sanitized to be able to compare with hf.
50 + """
51 + images = [asset.pil_image for asset in image_assets]
52 +
53 + # Export the images to a tempdir and substitute it into the hf prompt;
54 + # the contents between <img>/</img> will be ignored by VLLM, but the
55 + # transformers implementation for the visual transformer parses this to
56 + # reload it in the forward call; the contents are treated as a URL or a
57 + # local path.
58 + for idx, asset in enumerate(image_assets):
59 + image_tmp_path = tmp_path / f"{asset.name}.jpg"
60 + asset.pil_image.save(image_tmp_path)
61 + HF_IMAGE_PROMPTS[idx] = HF_IMAGE_PROMPTS[idx].replace(
62 + "<img></img>", f"<img>{image_tmp_path}</img>")
63 +
64 + inputs_per_image = [(
65 + [prompt for _ in size_factors],
66 + [rescale_image_size(image, factor) for factor in size_factors],
67 + ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
68 +
69 + # NOTE: take care of the order. run vLLM first, and then run HF.
70 + # vLLM needs a fresh new process without cuda initialization.
71 + # if we run HF first, the cuda initialization will be done and it
72 + # will hurt multiprocessing backend with fork method (the default method).
73 +
74 + # max_model_len should be greater than image_feature_size
75 + # Qwen encodes images into a fixed content size of 256
76 + with vllm_runner(model,
77 + max_model_len=300,
78 + max_num_seqs=1,
79 + dtype=dtype,
80 + tensor_parallel_size=tensor_parallel_size,
81 + distributed_executor_backend=distributed_executor_backend,
82 + enforce_eager=True) as vllm_model:
83 + vllm_outputs_per_image = [
84 + vllm_model.generate_greedy_logprobs(prompts,
85 + max_tokens,
86 + num_logprobs=num_logprobs,
87 + images=images)
88 + for prompts, images in inputs_per_image
89 + ]
90 +
91 with hf_runner(model, dtype=dtype) as hf_model:
30 - hf_outputs = hf_model.generate_greedy_logprobs_limit(
31 - example_prompts,
32 - max_tokens,
33 - num_logprobs=num_logprobs,
92 + hf_outputs_per_image = [
93 + hf_model.generate_greedy_logprobs_limit(prompts,
94 + max_tokens,
95 + num_logprobs=num_logprobs,
96 + images=images)
97 + for prompts, images in inputs_per_image
98 + ]
99 +
100 + for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
101 + vllm_outputs_per_image):
102 +
103 + check_logprobs_close(
104 + outputs_0_lst=hf_outputs,
105 + outputs_1_lst=vllm_outputs,
106 + name_0="hf",
107 + name_1="vllm",
108 )
109
110 +
111 +@pytest.mark.parametrize("model", multimodal_models)
112 +@pytest.mark.parametrize(
113 + "size_factors",
114 + [
115 + # No image
116 + [],
117 + # Single-scale
118 + [1.0],
119 + # Single-scale, batched
120 + [1.0, 1.0, 1.0],
121 + # Multi-scale
122 + [0.25, 0.5, 1.0],
123 + ],
124 +)
125 +@pytest.mark.parametrize("dtype", ["bfloat16"])
126 +@pytest.mark.parametrize("max_tokens", [8])
127 +@pytest.mark.parametrize("num_logprobs", [5])
128 +def test_multimodal_models(tmp_path, hf_runner, vllm_runner, image_assets,
129 + model, size_factors, dtype, max_tokens,
130 + num_logprobs) -> None:
131 + run_test(
132 + tmp_path,
133 + hf_runner,
134 + vllm_runner,
135 + image_assets,
136 + model,
137 + size_factors=size_factors,
138 + dtype=dtype,
139 + max_tokens=max_tokens,
140 + num_logprobs=num_logprobs,
141 + tensor_parallel_size=1,
142 + )
143 +
144 +
145 +# Ensure that a text-only Qwen model can still be loaded and
146 +# used for inference in VLLM without throwing.
147 +@pytest.mark.parametrize("model", text_only_models)
148 +@pytest.mark.parametrize("dtype", ["bfloat16"])
149 +@pytest.mark.parametrize("max_tokens", [32])
150 +@pytest.mark.parametrize("num_logprobs", [5])
151 +def test_text_only_qwen_model_can_be_loaded_and_run(
152 + vllm_runner: Type[VllmRunner],
153 + example_prompts,
154 + model: str,
155 + *,
156 + dtype: str,
157 + max_tokens: int,
158 + num_logprobs: int,
159 +):
160 with vllm_runner(model, dtype=dtype) as vllm_model:
37 - vllm_outputs = vllm_model.generate_greedy_logprobs(
161 + vllm_model.generate_greedy_logprobs(
162 example_prompts,
163 max_tokens,
164 num_logprobs=num_logprobs,
165 )
42 -
43 - check_logprobs_close(
44 - outputs_0_lst=hf_outputs,
45 - outputs_1_lst=vllm_outputs,
46 - name_0="hf",
47 - name_1="vllm",
48 - )
vllm/entrypoints/chat_utils.py
+2
@@ -150,6 +150,8 @@ class BaseMultiModalItemTracker(ABC, Generic[_T]):
150 if model_type in ("blip-2", "chatglm", "fuyu", "paligemma"):
151 # These models do not use image tokens in the prompt
152 return None
153 + if model_type == "qwen":
154 + return f"Picture {current_count}: <img></img>"
155 if model_type.startswith("llava"):
156 return self._cached_token_str(self._tokenizer,
157 hf_config.image_token_index)
vllm/model_executor/layers/resampler.py new
+273
@@ -0,0 +1,273 @@
1 +# coding=utf-8
2 +# Adapted from
3 +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
4 +# https://huggingface.co/Qwen/Qwen-7B/blob/main/modeling_qwen.py
5 +# https://github.com/facebookresearch/mae/blob/efb2a8062c206524e35e47d04501ed4f544c0ae8/util/pos_embed.py#L20
6 +#
7 +# Copyright 2023 The Qwen team.
8 +# Copyright 2023 The vLLM team.
9 +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
10 +#
11 +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
12 +# and OPT implementations in this library. It has been modified from its
13 +# original forms to accommodate minor architectural differences compared
14 +# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
15 +#
16 +# Licensed under the Apache License, Version 2.0 (the "License");
17 +# you may not use this file except in compliance with the License.
18 +# You may obtain a copy of the License at
19 +#
20 +# http://www.apache.org/licenses/LICENSE-2.0
21 +#
22 +# Unless required by applicable law or agreed to in writing, software
23 +# distributed under the License is distributed on an "AS IS" BASIS,
24 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
25 +# See the License for the specific language governing permissions and
26 +# limitations under the License.
27 +"""
28 +Shared resampler perceiver network used in multimodal models and
29 +related helpers for sincos positional embeddings.
30 +
31 +Example models: Qwen (Qwen-VL), Minicpmv2.0
32 +"""
33 +import math
34 +from functools import partial
35 +from typing import Callable, Optional, Tuple, Union
36 +
37 +import numpy as np
38 +import torch
39 +import torch.nn.functional as F
40 +from torch import nn
41 +from torch.nn.init import trunc_normal_
42 +
43 +from vllm.model_executor.layers.linear import ReplicatedLinear
44 +
45 +DEFAULT_LN = partial(nn.LayerNorm, eps=1e-6)
46 +
47 +
48 +def get_abs_pos(abs_pos: torch.Tensor, tgt_size: Union[torch.Tensor,
49 + int]) -> torch.Tensor:
50 + # abs_pos: L, C
51 + # tgt_size: (H, W)
52 + # return: M, C
53 + src_size = int(math.sqrt(abs_pos.size(0)))
54 + dtype = abs_pos.dtype
55 + if isinstance(tgt_size, int):
56 + tgt_size = (tgt_size, tgt_size)
57 + if (src_size == tgt_size[0] and src_size == tgt_size[1]):
58 + return abs_pos
59 + return (F.interpolate(
60 + abs_pos.float().reshape(1, src_size, src_size, -1).permute(0, 3, 1, 2),
61 + size=(tgt_size[0], tgt_size[1]),
62 + mode="bicubic",
63 + align_corners=False,
64 + ).permute(0, 2, 3, 1).flatten(0, 2).to(dtype=dtype))
65 +
66 +
67 +# sin/cos positional embedding helpers are adapted from:
68 +# https://github.com/facebookresearch/mae/blob/efb2a8062c206524e35e47d04501ed4f544c0ae8/util/pos_embed.py#L20
69 +def get_1d_sincos_pos_embed_from_grid(
70 + embed_dim: int, pos: np.ndarray,
71 + version: Tuple[int, int] = (2, 0)) -> torch.Tensor:
72 + """
73 + embed_dim: output dimension for each position
74 + pos: a list of positions to be encoded: size (M,) / (H, W)
75 + out: (M, D) / (H, W, D)
76 + """
77 + assert embed_dim % 2 == 0
78 + omega = np.arange(embed_dim // 2, dtype=np.float32)
79 + omega /= embed_dim / 2.0
80 + omega = 1.0 / 10000**omega # (D/2,)
81 +
82 + if version == (2, 0):
83 + pos = pos.reshape(-1) # (M,)
84 + out = np.einsum("m,d->md", pos, omega) # (M, D/2), outer product
85 + emb_sin = np.sin(out) # (M, D/2)
86 + emb_cos = np.cos(out) # (M, D/2)
87 + emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D)
88 + else:
89 + out = np.einsum("hw,d->hwd", pos, omega) # (H, W, D/2), outer product
90 + emb_sin = np.sin(out) # (H, W, D/2)
91 + emb_cos = np.cos(out) # (H, W, D/2)
92 + emb = np.concatenate([emb_sin, emb_cos], axis=-1) # (H, W, D)
93 + return emb
94 +
95 +
96 +def get_2d_sincos_pos_embed_from_grid(
97 + embed_dim: int, grid: np.ndarray,
98 + version: Tuple[int, int] = (2, 0)) -> torch.Tensor:
99 + assert embed_dim % 2 == 0
100 +
101 + # use half of dimensions to encode grid_h
102 + emb_h = get_1d_sincos_pos_embed_from_grid(
103 + embed_dim // 2, grid[0], version) # (H*W, D/2) or (H, W, D/2)
104 + emb_w = get_1d_sincos_pos_embed_from_grid(
105 + embed_dim // 2, grid[1], version) # (H*W, D/2) or (H, W, D/2)
106 +
107 + if version == (2, 0):
108 + emb = np.concatenate([emb_h, emb_w], axis=1) # (H*W, D)
109 + else:
110 + emb = np.concatenate([emb_h, emb_w], axis=-1) # (H, W, D)
111 + return emb
112 +
113 +
114 +def get_2d_sincos_pos_embed(
115 + embed_dim: int,
116 + grid_size: Union[int, Tuple[int, int]],
117 + cls_token: bool = False,
118 + version: Tuple[int, int] = (2, 0),
119 +) -> torch.Tensor:
120 + """
121 + grid_size: int of the grid height and width
122 + return:
123 + pos_embed: [grid_size*grid_size, embed_dim] or
124 + [1+grid_size*grid_size, embed_dim] (w/ or w/o cls_token)
125 + """
126 + if isinstance(grid_size, int):
127 + grid_h_size, grid_w_size = grid_size, grid_size
128 + else:
129 + grid_h_size, grid_w_size = grid_size[0], grid_size[1]
130 +
131 + grid_h = np.arange(grid_h_size, dtype=np.float32)
132 + grid_w = np.arange(grid_w_size, dtype=np.float32)
133 + grid = np.meshgrid(grid_w, grid_h) # here w goes first
134 + grid = np.stack(grid, axis=0)
135 + assert isinstance(grid, np.ndarray) and \
136 + grid.shape == (2, grid_h_size, grid_w_size)
137 +
138 + if version == (2, 0):
139 + grid = grid.reshape([2, 1, grid_h_size, grid_w_size])
140 + pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid, version)
141 + if cls_token:
142 + pos_embed = np.concatenate([np.zeros([1, embed_dim]), pos_embed],
143 + axis=0)
144 + else:
145 + pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid, version)
146 + return pos_embed
147 +
148 +
149 +class BaseResampler(nn.Module):
150 + """
151 + A 2D perceiver-resampler network with one cross attention layers by
152 + (grid_size**2) learnable queries and 2d sincos pos_emb.
153 + Outputs:
154 + A tensor with the shape of (grid_size**2, embed_dim)
155 + """
156 +
157 + def __init__(
158 + self,
159 + num_queries: int,
160 + embed_dim: int,
161 + num_heads: int,
162 + kv_dim: Optional[int] = None,
163 + norm_layer: Callable[[int], nn.LayerNorm] = DEFAULT_LN,
164 + do_post_projection: bool = True,
165 + ) -> None:
166 + super().__init__()
167 +
168 + self.num_queries = num_queries
169 + self.embed_dim = embed_dim
170 + self.num_heads = num_heads
171 +
172 + self.query = nn.Parameter(torch.zeros(self.num_queries, embed_dim))
173 + trunc_normal_(self.query, std=0.02)
174 + if kv_dim is not None and kv_dim != embed_dim:
175 + self.kv_proj = ReplicatedLinear(kv_dim, embed_dim, bias=False)
176 + else:
177 + # Maintain the same return value with ReplicatedLinear.forward
178 + self.kv_proj = lambda *args, **kwargs: ( # type: ignore # noqa
179 + nn.Identity()(*args, **kwargs),
180 + None,
181 + )
182 + self.attn = nn.MultiheadAttention(embed_dim, num_heads)
183 + self.ln_q = norm_layer(embed_dim)
184 + self.ln_kv = norm_layer(embed_dim)
185 + self.do_post_projection = do_post_projection
186 + self.ln_post = norm_layer(embed_dim) if do_post_projection else None
187 + self.proj = nn.Parameter(
188 + (embed_dim**-0.5) *
189 + torch.randn(embed_dim, embed_dim)) if do_post_projection else None
190 +
191 + def _init_weights(self, m: nn.Module) -> None:
192 + if isinstance(m, nn.Linear):
193 + trunc_normal_(m.weight, std=0.02)
194 + if isinstance(m, nn.Linear) and m.bias is not None:
195 + nn.init.constant_(m.bias, 0)
196 + elif isinstance(m, nn.LayerNorm):
197 + nn.init.constant_(m.bias, 0)
198 + nn.init.constant_(m.weight, 1.0)
199 +
200 + def _repeat(self, query, N: int):
201 + return query.unsqueeze(1).repeat(1, N, 1)
202 +
203 +
204 +class Resampler2(BaseResampler):
205 + """Resampler-perceiver network to be used for a variety of model types,
206 + e.g., Qwen-vl / Minicpmv 2.0. The main difference is the addition of the
207 + do_post_projection arg, which indicates whether or not there should be
208 + a post layer normalization and projector after the attention. This is
209 + present in minicpmv2.0, but not qwen-vl.
210 + """
211 +
212 + def __init__(
213 + self,
214 + grid_size: int,
215 + embed_dim: int,
216 + num_heads: int,
217 + kv_dim: Optional[int] = None,
218 + norm_layer: Callable[[int], nn.LayerNorm] = DEFAULT_LN,
219 + adaptive: bool = False,
220 + do_post_projection: bool = True,
221 + ) -> None:
222 + super().__init__(grid_size**2,
223 + embed_dim,
224 + num_heads,
225 + kv_dim,
226 + norm_layer,
227 + do_post_projection=do_post_projection)
228 +
229 + self.adaptive = adaptive
230 + pos_embed_arr = get_2d_sincos_pos_embed(embed_dim,
231 + grid_size,
232 + version=(2, 0))
233 +
234 + self.pos_embed = nn.Parameter(
235 + torch.from_numpy(pos_embed_arr).requires_grad_(False))
236 +
237 + self.apply(self._init_weights)
238 +
239 + def forward(
240 + self,
241 + x: torch.Tensor,
242 + tgt_sizes: Optional[torch.Tensor] = None,
243 + attn_mask: Optional[torch.Tensor] = None,
244 + ) -> torch.Tensor:
245 + if tgt_sizes is None:
246 + tgt_sizes = int(math.sqrt(x.size(1)))
247 + if self.adaptive:
248 + pos_embed_arr = get_2d_sincos_pos_embed(self.embed_dim,
249 + tgt_sizes,
250 + version=(2, 0))
251 + pos_embed = torch.from_numpy(pos_embed_arr).to(device=x.device,
252 + dtype=x.dtype)
253 + else:
254 + pos_embed = get_abs_pos(self.pos_embed,
255 + tgt_sizes).to(device=x.device,
256 + dtype=x.dtype)
257 +
258 + x, _ = self.kv_proj(x)
259 + x = self.ln_kv(x).permute(1, 0, 2)
260 +
261 + N = x.shape[1]
262 + q = self.ln_q(self.query)
263 + out = self.attn(
264 + self._repeat(q, N) + self.pos_embed.unsqueeze(1),
265 + x + pos_embed.unsqueeze(1),
266 + x,
267 + attn_mask=attn_mask,
268 + )[0]
269 + x = out.permute(1, 0, 2)
270 + if self.do_post_projection:
271 + x = self.ln_post(x)
272 + x = x @ self.proj
273 + return x
vllm/model_executor/models/__init__.py
+1 -1
@@ -51,7 +51,6 @@ _GENERATION_MODELS = {
51 "PhiForCausalLM": ("phi", "PhiForCausalLM"),
52 "Phi3ForCausalLM": ("llama", "LlamaForCausalLM"),
53 "PhiMoEForCausalLM": ("phimoe", "PhiMoEForCausalLM"),
54 - "QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
54 "Qwen2ForCausalLM": ("qwen2", "Qwen2ForCausalLM"),
55 "Qwen2MoeForCausalLM": ("qwen2_moe", "Qwen2MoeForCausalLM"),
56 "RWForCausalLM": ("falcon", "FalconForCausalLM"),
@@ -88,6 +87,7 @@ _MULTIMODAL_MODELS = {
87 "PaliGemmaForConditionalGeneration"),
88 "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
89 "UltravoxModel": ("ultravox", "UltravoxModel"),
90 + "QWenLMHeadModel": ("qwen", "QWenLMHeadModel"),
91 }
92 _CONDITIONAL_GENERATION_MODELS = {
93 "BartModel": ("bart", "BartForConditionalGeneration"),
vllm/model_executor/models/minicpmv.py
+5 -155
@@ -26,11 +26,9 @@ import re
26 from array import array
27 from functools import partial
28 from typing import (Any, Callable, Iterable, List, Mapping, Optional, Tuple,
29 - TypedDict, Union)
29 + TypedDict)
30
31 -import numpy as np
31 import torch
33 -import torch.nn.functional as F
32 import torch.types
33 from PIL import Image
34 from torch import nn
@@ -44,6 +42,8 @@ from vllm.logger import init_logger
42 from vllm.model_executor.layers.linear import ReplicatedLinear
43 from vllm.model_executor.layers.logits_processor import LogitsProcessor
44 from vllm.model_executor.layers.quantization import QuantizationConfig
45 +from vllm.model_executor.layers.resampler import (Resampler2,
46 + get_2d_sincos_pos_embed)
47 from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
48 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
49 from vllm.model_executor.model_loader.utils import set_default_torch_dtype
@@ -98,101 +98,6 @@ MiniCPMVImageInputs = MiniCPMVImagePixelInputs
98 DEFAULT_LN = partial(nn.LayerNorm, eps=1e-6)
99
100
101 -def get_abs_pos(abs_pos: torch.Tensor, tgt_size: torch.Tensor):
102 - # abs_pos: L, C
103 - # tgt_size: (H, W)
104 - # return: M, C
105 - src_size = int(math.sqrt(abs_pos.size(0)))
106 - # tgt_size = int(math.sqrt(tgt_size))
107 - dtype = abs_pos.dtype
108 -
109 - return (F.interpolate(
110 - abs_pos.float().reshape(1, src_size, src_size, -1).permute(0, 3, 1, 2),
111 - size=(tgt_size[0], tgt_size[1]),
112 - mode="bicubic",
113 - align_corners=False,
114 - ).permute(0, 2, 3, 1).flatten(0, 2).to(dtype=dtype))
115 -
116 -
117 -# https://github.com/facebookresearch/mae/blob/efb2a8062c206524e35e47d04501ed4f544c0ae8/util/pos_embed.py#L20
118 -def get_2d_sincos_pos_embed(
119 - embed_dim: int,
120 - grid_size: Union[int, Tuple[int, int]],
121 - cls_token: bool = False,
122 - version: Tuple[int, int] = (2, 0),
123 -):
124 - """
125 - grid_size: int of the grid height and width
126 - return:
127 - pos_embed: [grid_size*grid_size, embed_dim] or
128 - [1+grid_size*grid_size, embed_dim] (w/ or w/o cls_token)
129 - """
130 - if isinstance(grid_size, int):
131 - grid_h_size, grid_w_size = grid_size, grid_size
132 - else:
133 - grid_h_size, grid_w_size = grid_size[0], grid_size[1]
134 -
135 - grid_h = np.arange(grid_h_size, dtype=np.float32)
136 - grid_w = np.arange(grid_w_size, dtype=np.float32)
137 - grid = np.meshgrid(grid_w, grid_h) # here w goes first
138 - grid = np.stack(grid, axis=0)
139 -
140 - if version == (2, 0):
141 - grid = grid.reshape([2, 1, grid_h_size, grid_w_size])
142 - pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid, version)
143 - if cls_token:
144 - pos_embed = np.concatenate([np.zeros([1, embed_dim]), pos_embed],
145 - axis=0)
146 - else:
147 - pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid, version)
148 - return pos_embed
149 -
150 -
151 -def get_2d_sincos_pos_embed_from_grid(embed_dim: int,
152 - grid: np.ndarray,
153 - version: Tuple[int, int] = (2, 0)):
154 - assert embed_dim % 2 == 0
155 -
156 - # use half of dimensions to encode grid_h
157 - emb_h = get_1d_sincos_pos_embed_from_grid(
158 - embed_dim // 2, grid[0], version) # (H*W, D/2) or (H, W, D/2)
159 - emb_w = get_1d_sincos_pos_embed_from_grid(
160 - embed_dim // 2, grid[1], version) # (H*W, D/2) or (H, W, D/2)
161 -
162 - if version == (2, 0):
163 - emb = np.concatenate([emb_h, emb_w], axis=1) # (H*W, D)
164 - else:
165 - emb = np.concatenate([emb_h, emb_w], axis=-1) # (H, W, D)
166 - return emb
167 -
168 -
169 -def get_1d_sincos_pos_embed_from_grid(embed_dim: int,
170 - pos: np.ndarray,
171 - version: Tuple[int, int] = (2, 0)):
172 - """
173 - embed_dim: output dimension for each position
174 - pos: a list of positions to be encoded: size (M,) / (H, W)
175 - out: (M, D) / (H, W, D)
176 - """
177 - assert embed_dim % 2 == 0
178 - omega = np.arange(embed_dim // 2, dtype=np.float32)
179 - omega /= embed_dim / 2.0
180 - omega = 1.0 / 10000**omega # (D/2,)
181 -
182 - if version == (2, 0):
183 - pos = pos.reshape(-1) # (M,)
184 - out = np.einsum("m,d->md", pos, omega) # (M, D/2), outer product
185 - emb_sin = np.sin(out) # (M, D/2)
186 - emb_cos = np.cos(out) # (M, D/2)
187 - emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D)
188 - else:
189 - out = np.einsum("hw,d->hwd", pos, omega) # (H, W, D/2), outer product
190 - emb_sin = np.sin(out) # (H, W, D/2)
191 - emb_cos = np.cos(out) # (H, W, D/2)
192 - emb = np.concatenate([emb_sin, emb_cos], axis=-1) # (H, W, D)
193 - return emb
194 -
195 -
101 class BaseResampler(nn.Module):
102 """
103 A 2D perceiver-resampler network with one cross attention layers by
@@ -245,62 +150,6 @@ class BaseResampler(nn.Module):
150 return query.unsqueeze(1).repeat(1, N, 1)
151
152
248 -class Resampler2(BaseResampler):
249 -
250 - def __init__(
251 - self,
252 - grid_size: int,
253 - embed_dim: int,
254 - num_heads: int,
255 - kv_dim: Optional[int] = None,
256 - norm_layer: Callable[[int], nn.LayerNorm] = DEFAULT_LN,
257 - adaptive: bool = False,
258 - ) -> None:
259 - super().__init__(grid_size**2, embed_dim, num_heads, kv_dim,
260 - norm_layer)
261 -
262 - self.adaptive = adaptive
263 - pos_embed_arr = get_2d_sincos_pos_embed(embed_dim,
264 - grid_size,
265 - version=(2, 0))
266 - self.pos_embed = nn.Parameter(
267 - torch.from_numpy(pos_embed_arr).float()).requires_grad_(False)
268 -
269 - self.apply(self._init_weights)
270 -
271 - def forward(
272 - self,
273 - x: torch.Tensor,
274 - tgt_sizes: torch.Tensor,
275 - attn_mask: Optional[torch.Tensor] = None,
276 - ):
277 - if self.adaptive:
278 - pos_embed_arr = get_2d_sincos_pos_embed(self.embed_dim,
279 - tgt_sizes,
280 - version=(2, 0))
281 - pos_embed = torch.from_numpy(pos_embed_arr).to(device=x.device,
282 - dtype=x.dtype)
283 - else:
284 - pos_embed = get_abs_pos(self.pos_embed, tgt_sizes)
285 -
286 - x, _ = self.kv_proj(x)
287 - x = self.ln_kv(x).permute(1, 0, 2)
288 -
289 - N = x.shape[1]
290 - q = self.ln_q(self.query)
291 - out = self.attn(
292 - self._repeat(q, N) + self.pos_embed.unsqueeze(1),
293 - x + pos_embed.unsqueeze(1),
294 - x,
295 - attn_mask=attn_mask,
296 - )[0]
297 - x = out.permute(1, 0, 2)
298 -
299 - x = self.ln_post(x)
300 - x = x @ self.proj
301 - return x
302 -
303 -
153 class Resampler2_5(BaseResampler):
154
155 def __init__(
@@ -782,7 +631,8 @@ class MiniCPMV2_0(MiniCPMVBaseModel):
631 num_heads=embed_dim // 128,
632 grid_size=int(math.sqrt(self.config.query_num)),
633 kv_dim=vision_dim,
785 - adaptive=True,
634 + adaptive=False,
635 + do_post_projection=True,
636 )
637
638 return resampler
vllm/model_executor/models/qwen.py
+667 -27
@@ -4,36 +4,402 @@
4 # Copyright (c) Alibaba Cloud.
5 # LICENSE: https://huggingface.co/Qwen/Qwen-7B/blob/main/LICENSE
6 """Inference-only QWen model compatible with HuggingFace weights."""
7 -from typing import Any, Dict, Iterable, List, Optional, Tuple
7
8 +import math
9 +import re
10 +from array import array
11 +from functools import partial
12 +from typing import (Any, Callable, Dict, Iterable, List, Literal, Mapping,
13 + Optional, Tuple, TypedDict, Union)
14 +
15 +import numpy as np
16 import torch
17 +from PIL import Image
18 from torch import nn
19 +from torchvision import transforms
20 +from torchvision.transforms import InterpolationMode
21 from transformers import PretrainedConfig
22
23 from vllm.attention import Attention, AttentionMetadata
14 -from vllm.config import CacheConfig
24 +from vllm.config import CacheConfig, MultiModalConfig
25 from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
16 -from vllm.model_executor.layers.activation import SiluAndMul
26 +from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
27 +from vllm.logger import init_logger
28 +from vllm.model_executor.layers.activation import SiluAndMul, get_act_fn
29 from vllm.model_executor.layers.layernorm import RMSNorm
18 -from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
30 +from vllm.model_executor.layers.linear import (ColumnParallelLinear,
31 + MergedColumnParallelLinear,
32 QKVParallelLinear,
33 RowParallelLinear)
34 from vllm.model_executor.layers.logits_processor import LogitsProcessor
35 from vllm.model_executor.layers.quantization.base_config import (
36 QuantizationConfig)
37 +from vllm.model_executor.layers.resampler import Resampler2, get_abs_pos
38 from vllm.model_executor.layers.rotary_embedding import get_rope
39 from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
40 from vllm.model_executor.layers.vocab_parallel_embedding import (
41 ParallelLMHead, VocabParallelEmbedding)
42 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
43 +from vllm.model_executor.models.interfaces import SupportsMultiModal
44 from vllm.model_executor.sampling_metadata import SamplingMetadata
30 -from vllm.sequence import IntermediateTensors
31 -from vllm.utils import print_warning_once
45 +from vllm.multimodal import MULTIMODAL_REGISTRY
46 +from vllm.multimodal.base import MultiModalInputs
47 +from vllm.multimodal.utils import cached_get_tokenizer
48 +from vllm.sequence import (VLLM_TOKEN_ID_ARRAY_TYPE, IntermediateTensors,
49 + SequenceData)
50 +
51 +from .utils import flatten_bn, is_pp_missing_parameter, make_layers
52 +
53 +logger = init_logger(__name__)
54 +
55 +# NOTE: Qwen models have a few other special tags, e.g., ref, bbox, quad;
56 +# for the time being, these tags are not considered as special at encoding
57 +# time. This may change as VLLMs multimodal API changes in the future.
58 +IMG_START = "<img>"
59 +IMG_END = "</img>"
60 +IMG_PAD = "<imgpad>"
61 +# Image context is fixed at 256 for all images
62 +MAX_QWEN_IMG_TOKENS = 256
63 +# Image normalization params
64 +CLIP_MEAN = (0.48145466, 0.4578275, 0.40821073)
65 +CLIP_STD = (0.26862954, 0.26130258, 0.27577711)
66 +
67 +
68 +class QwenImagePixelInputs(TypedDict):
69 + type: Literal["pixel_values"]
70 + data: torch.Tensor
71 + """
72 + Shape: `(batch_size * num_images, 3, image_size, image_size)`
73 +
74 + Note that image_size is the value in the vision config to which we resize
75 + the image to in the normalization transform. Currently multi-image support
76 + can only be leveraged by passing image embeddings directly.
77 + """
78 +
79 +
80 +class QwenImageEmbeddingInputs(TypedDict):
81 + type: Literal["image_embeds"]
82 + data: torch.Tensor
83 + """Shape: `(batch_size * num_images, 256, hidden_size)`
84 +
85 + `hidden_size` must match the hidden size of the language model backbone
86 + and is stored in the visual config of the model if we have one.
87 + """
88 +
89 +
90 +QwenImageInputs = Union[QwenImagePixelInputs, QwenImageEmbeddingInputs]
91 +
92 +
93 +class VisualAttention(nn.Module):
94 + """self-attention layer class.
95 + Self-attention layer takes input with size [s, b, h]
96 + and returns output of the same size.
97 + """
98 +
99 + def __init__(
100 + self,
101 + embed_dim: int,
102 + num_heads: int,
103 + bias: bool = True,
104 + kdim: Optional[int] = None,
105 + vdim: Optional[int] = None,
106 + ):
107 + super().__init__()
108 + self.embed_dim = embed_dim
109 + self.kdim = kdim if kdim is not None else embed_dim
110 + self.vdim = vdim if vdim is not None else embed_dim
111 + self._qkv_same_embed_dim = self.kdim == embed_dim \
112 + and self.vdim == embed_dim
113 +
114 + self.num_heads = num_heads
115 +
116 + # Per attention head and per partition values.
117 + assert embed_dim % num_heads == 0
118 + self.hidden_size_per_attention_head = embed_dim // num_heads
119 + self.num_attention_heads_per_partition = num_heads
120 + self.hidden_size_per_partition = embed_dim
121 +
122 + # Strided linear layer.
123 + assert self._qkv_same_embed_dim, \
124 + 'Visual Attention implementation only supports self-attention'
125 + self.in_proj = nn.Linear(embed_dim, 3 * embed_dim)
126 + self.out_proj = nn.Linear(embed_dim, embed_dim)
127 + self.norm_factor = math.sqrt(self.hidden_size_per_attention_head)
128 +
129 + def forward(
130 + self,
131 + x: torch.Tensor,
132 + attn_mask: Optional[torch.Tensor] = None,
133 + ) -> torch.Tensor:
134 + # query/key/value: [sq, b, h]
135 + sq, b, _ = x.size()
136 + mixed_x_layer = self.in_proj(x)
137 +
138 + # [sq, b, (np * 3 * hn)] --> [sq, b, np, 3 * hn]
139 + new_tensor_shape = mixed_x_layer.size()[:-1] + \
140 + (self.num_attention_heads_per_partition,
141 + 3 * self.hidden_size_per_attention_head)
142 + mixed_x_layer = mixed_x_layer.view(*new_tensor_shape)
143 +
144 + # [sq, b, np, 3 * hn] --> 3 [sq, b, np, hn]
145 + query_layer, key_layer, value_layer = mixed_x_layer.split(
146 + self.hidden_size_per_attention_head, dim=-1)
147 +
148 + # [sq, b, np, hn] -> [sq, b * np, hn]
149 + query_layer = query_layer.view(
150 + sq, b * self.num_attention_heads_per_partition,
151 + self.hidden_size_per_attention_head).transpose(0, 1)
152 + # [sk, b, np, hn] -> [sk, b * np, hn]
153 + key_layer = key_layer.view(
154 + sq, b * self.num_attention_heads_per_partition,
155 + self.hidden_size_per_attention_head).transpose(0, 1)
156 +
157 + q_scaled = query_layer / self.norm_factor
158 + if attn_mask is not None:
159 + attention_probs = torch.baddbmm(attn_mask, q_scaled,
160 + key_layer.transpose(-2, -1))
161 + else:
162 + attention_probs = torch.bmm(q_scaled, key_layer.transpose(-2, -1))
163 + attention_probs = attention_probs.softmax(dim=-1)
164 +
165 + value_layer = value_layer.view(
166 + sq, b * self.num_attention_heads_per_partition,
167 + self.hidden_size_per_attention_head).transpose(0, 1)
168 +
169 + # matmul: [b * np, sq, hn]
170 + context_layer = torch.bmm(attention_probs, value_layer)
171
33 -from .utils import is_pp_missing_parameter, make_layers
172 + # change view [b, np, sq, hn]
173 + context_layer = context_layer.view(
174 + b, self.num_attention_heads_per_partition, sq,
175 + self.hidden_size_per_attention_head)
176 +
177 + # [b, np, sq, hn] --> [sq, b, np, hn]
178 + context_layer = context_layer.permute(2, 0, 1, 3).contiguous()
179 +
180 + # [sq, b, np, hn] --> [sq, b, hp]
181 + new_context_layer_shape = context_layer.size()[:-2] + \
182 + (self.hidden_size_per_partition,)
183 + context_layer = context_layer.view(*new_context_layer_shape)
184 +
185 + output = self.out_proj(context_layer)
186 +
187 + return output
188 +
189 +
190 +class QwenVMLP(nn.Module):
191 + """MLP for the visual component of the Qwen model."""
192 +
193 + def __init__(
194 + self,
195 + hidden_size: int,
196 + intermediate_size: int,
197 + quant_config: Optional[QuantizationConfig] = None,
198 + ):
199 + super().__init__()
200 + self.c_fc = ColumnParallelLinear(hidden_size,
201 + intermediate_size,
202 + bias=True,
203 + quant_config=quant_config)
204 + self.act_fn = get_act_fn("gelu", quant_config, intermediate_size)
205 + self.c_proj = RowParallelLinear(
206 + intermediate_size,
207 + hidden_size,
208 + bias=True,
209 + quant_config=quant_config,
210 + )
211 +
212 + def forward(self, x):
213 + x, _ = self.c_fc(x)
214 + x = self.act_fn(x)
215 + x, _ = self.c_proj(x)
216 + return x
217 +
218 +
219 +class VisualAttentionBlock(nn.Module):
220 +
221 + def __init__(
222 + self,
223 + d_model: int,
224 + n_head: int,
225 + mlp_ratio: float = 4.0,
226 + norm_layer: Callable = nn.LayerNorm,
227 + quant_config: Optional[QuantizationConfig] = None,
228 + ):
229 + super().__init__()
230 +
231 + self.ln_1 = norm_layer(d_model)
232 + self.ln_2 = norm_layer(d_model)
233 + mlp_width = int(d_model * mlp_ratio)
234 + self.attn = VisualAttention(d_model, n_head)
235 + self.mlp = QwenVMLP(
236 + hidden_size=d_model,
237 + intermediate_size=mlp_width,
238 + quant_config=quant_config,
239 + )
240 +
241 + def attention(
242 + self,
243 + x: torch.Tensor,
244 + attn_mask: Optional[torch.Tensor] = None,
245 + ) -> torch.Tensor:
246 + attn_mask = attn_mask.to(x.dtype) if attn_mask is not None else None
247 + return self.attn(x, attn_mask=attn_mask)
248 +
249 + def forward(
250 + self,
251 + x: torch.Tensor,
252 + attn_mask: Optional[torch.Tensor] = None,
253 + ) -> torch.Tensor:
254 + x = x + self.attention(self.ln_1(x), attn_mask=attn_mask)
255 + x = x + self.mlp(self.ln_2(x))
256 + return x
257 +
258 +
259 +class TransformerBlock(nn.Module):
260 +
261 + def __init__(
262 + self,
263 + width: int,
264 + layers: int,
265 + heads: int,
266 + mlp_ratio: float = 4.0,
267 + norm_layer: Callable = nn.LayerNorm,
268 + quant_config: Optional[QuantizationConfig] = None,
269 + ):
270 + super().__init__()
271 + self.width = width
272 + self.layers = layers
273 +
274 + self.resblocks = nn.ModuleList([
275 + VisualAttentionBlock(width,
276 + heads,
277 + mlp_ratio,
278 + norm_layer=norm_layer,
279 + quant_config=quant_config)
280 + for _ in range(layers)
281 + ])
282 +
283 + def get_cast_dtype(self) -> torch.dtype:
284 + return self.resblocks[0].mlp.c_fc.weight.dtype
285 +
286 + def get_cast_device(self) -> torch.device:
287 + return self.resblocks[0].mlp.c_fc.weight.device
288 +
289 + def forward(self,
290 + x: torch.Tensor,
291 + attn_mask: Optional[torch.Tensor] = None) -> torch.Tensor:
292 + for r in self.resblocks:
293 + x = r(x, attn_mask=attn_mask)
294 + return x
295 +
296 +
297 +class VisionTransformer(nn.Module):
298 +
299 + def __init__(self,
300 + image_size: int,
301 + patch_size: int,
302 + width: int,
303 + layers: int,
304 + heads: int,
305 + mlp_ratio: float,
306 + n_queries: int = 256,
307 + output_dim: int = 512,
308 + image_start_id: int = 151857,
309 + quant_config: Optional[QuantizationConfig] = None,
310 + **kwargs):
311 + super().__init__()
312 + image_height, image_width = self.image_size = (image_size, image_size)
313 + patch_height, patch_width = self.patch_size = (patch_size, patch_size)
314 + self.grid_size = (image_height // patch_height,
315 + image_width // patch_width)
316 + self.output_dim = output_dim
317 + self.conv1 = nn.Conv2d(in_channels=3,
318 + out_channels=width,
319 + kernel_size=patch_size,
320 + stride=patch_size,
321 + bias=False)
322 +
323 + # class embeddings and positional embeddings
324 + scale = width**-0.5
325 + self.positional_embedding = nn.Parameter(scale *
326 + torch.randn(256, width))
327 +
328 + norm_layer = partial(nn.LayerNorm, eps=1e-6)
329 +
330 + self.ln_pre = norm_layer(width)
331 + self.transformer = TransformerBlock(width,
332 + layers,
333 + heads,
334 + mlp_ratio,
335 + norm_layer=norm_layer,
336 + quant_config=quant_config)
337 +
338 + self.attn_pool = Resampler2(
339 + grid_size=int(math.sqrt(n_queries)),
340 + embed_dim=output_dim,
341 + num_heads=output_dim // 128,
342 + kv_dim=width,
343 + norm_layer=norm_layer,
344 + adaptive=False,
345 + do_post_projection=False,
346 + ).to(
347 + device=self.positional_embedding.device,
348 + dtype=self.positional_embedding.dtype,
349 + )
350 +
351 + self.ln_post = norm_layer(output_dim)
352 + self.proj = nn.Parameter(
353 + (output_dim**-0.5) * torch.randn(output_dim, output_dim))
354 + self.image_start_id = image_start_id
355 + self.image_end_id = image_start_id + 1
356 +
357 + def forward(self, x: torch.Tensor) -> torch.Tensor:
358 + x = x.to(
359 + dtype=self.transformer.get_cast_dtype(),
360 + device=self.transformer.get_cast_device(),
361 + )
362 +
363 + # to patches
364 + x = self.conv1(x) # shape = [*, width, grid, grid]
365 + x = x.reshape(x.shape[0], x.shape[1],
366 + -1) # shape = [*, width, grid ** 2]
367 + x = x.permute(0, 2, 1) # shape = [*, grid ** 2, width]
368 +
369 + x = x + get_abs_pos(self.positional_embedding, int(math.sqrt(
370 + x.size(1))))
371 +
372 + x = self.ln_pre(x)
373 +
374 + x = x.permute(1, 0, 2) # NLD -> LND
375 + x = self.transformer(x)
376 + x = x.permute(1, 0, 2) # LND -> NLD
377 +
378 + x = self.attn_pool(x)
379 + x = self.ln_post(x)
380 + x = x @ self.proj
381 +
382 + return x
383 +
384 + def get_image_positions(self,
385 + input_ids: torch.Tensor) -> Optional[torch.Tensor]:
386 + """Given the input IDs, extracts start/stop points corresponding to
387 + images.
388 +
389 + args:
390 + Returns:
391 + Optional torch tensor corresponding to start/stop pairs of images.
392 + """
393 + if torch.any(input_ids == self.image_start_id):
394 + bos_pos = torch.where(input_ids == self.image_start_id)
395 + eos_pos = torch.where(input_ids == self.image_end_id)
396 + return torch.stack((bos_pos[0], eos_pos[0]), dim=1)
397 + return None
398
399
400 class QWenMLP(nn.Module):
401 + """MLP for the language component of the Qwen model, which contains a
402 + MergedColumnParallelLinear merging 2 outputs via silu activation."""
403
404 def __init__(
405 self,
@@ -56,7 +422,7 @@ class QWenMLP(nn.Module):
422 "Only silu is supported for now.")
423 self.act_fn = SiluAndMul()
424
59 - def forward(self, x):
425 + def forward(self, x: torch.Tensor) -> torch.Tensor:
426 gate_up, _ = self.gate_up_proj(x)
427 x = self.act_fn(gate_up)
428 x, _ = self.c_proj(x)
@@ -203,6 +569,9 @@ class QWenModel(nn.Module):
569 lambda prefix: QWenBlock(config, cache_config, quant_config),
570 prefix=f"{prefix}.h")
571 self.ln_f = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
572 + self.visual = VisionTransformer(**config.visual,
573 + quant_config=quant_config) if hasattr(
574 + config, "visual") else None
575
576 def forward(
577 self,
@@ -211,9 +580,33 @@ class QWenModel(nn.Module):
580 kv_caches: List[torch.Tensor],
581 attn_metadata: AttentionMetadata,
582 intermediate_tensors: Optional[IntermediateTensors],
583 + pixel_values: Optional[QwenImageInputs],
584 ) -> torch.Tensor:
585 + img_pos = None
586 + # If pixel / visual embeddings are provided, this is a visual model
587 + if pixel_values is not None and self.visual is not None:
588 + if pixel_values["type"] != "image_embeds":
589 + image_embeds = self.visual(pixel_values["data"])
590 + else:
591 + image_embeds = pixel_values["data"]
592 +
593 + # features should be of shape (# images, 256, hidden_dim)
594 + img_pos = self.visual.get_image_positions(input_ids)
595 + if isinstance(
596 + img_pos,
597 + np.ndarray) and img_pos.shape[0] != image_embeds.shape[0]:
598 + raise ValueError(
599 + f"Number of placeholders: {img_pos.shape[0]} "
600 + f"does not match number of images {image_embeds.shape[0]}."
601 + )
602 +
603 if get_pp_group().is_first_rank:
604 hidden_states = self.wte(input_ids)
605 + # Merge the image embeddings into the hidden states if actually have
606 + # visual features and the corresponding image tokens
607 + if img_pos is not None:
608 + for idx, (img_bos, img_eos) in enumerate(img_pos):
609 + hidden_states[img_bos + 1:img_eos] = image_embeds[idx]
610 residual = None
611 else:
612 assert intermediate_tensors is not None
@@ -237,16 +630,241 @@ class QWenModel(nn.Module):
630 return hidden_states
631
632
240 -class QWenLMHeadModel(nn.Module):
633 +def get_image_text(image_num: int, padding: bool) -> str:
634 + """Retrieves a placeholder text that when tokenized, will be expanded with
635 + image pads.
636 +
637 + Args:
638 + image_num: The number of the image that we want a text prompt for.
639 + Images should be indexed starting at 1.
640 + padding: Whether or not padding should be manually added.
641 +
642 + Returns:
643 + Text placeholder prompt for the image being considered.
644 + """
645 + image_start = f"Picture {image_num}: {IMG_START}"
646 + image_end = f"{IMG_END}\n"
647 + if not padding:
648 + return f"{image_start}{image_end}"
649 + return f"{image_start}{MAX_QWEN_IMG_TOKENS * IMG_PAD}{image_end}"
650 +
651 +
652 +def input_processor_for_qwen(ctx: InputContext,
653 + llm_inputs: LLMInputs) -> LLMInputs:
654 + """Processes the inputs, which may or may not be multimodal.
655 + Multimodal inputs will only be processed if the model has a "visual"
656 + component in its model config, otherwise they'll be ignored.
657 +
658 + Args:
659 + ctx: Context of the loaded model.
660 + llm_inputs: LLM inputs which may have a multi_modal_data attribute.
661 +
662 + Returns:
663 + If the model is language only or not multimodal inputs were provided,
664 + returns llm_inputs unmodified. Otherwise, processes the multimodal
665 + images / image embeddings and adds the fixed-length image placeholders.
666 + """
667 + multi_modal_data = llm_inputs.get("multi_modal_data")
668 +
669 + # Only process images if we have multimodal data and a visual config
670 + hf_config = ctx.get_hf_config()
671 + if (multi_modal_data is None or "image" not in multi_modal_data
672 + or not hasattr(hf_config, "visual")):
673 + return llm_inputs
674 +
675 + prompt = llm_inputs.get("prompt")
676 + prompt_token_ids = llm_inputs["prompt_token_ids"]
677 + model_config = ctx.model_config
678 + tokenizer = cached_get_tokenizer(model_config.tokenizer,
679 + trust_remote_code=True)
680 + image_data = multi_modal_data["image"]
681 + if isinstance(image_data, torch.Tensor):
682 + num_dims = len(image_data.shape)
683 + if num_dims < 2 or num_dims > 3:
684 + raise ValueError(
685 + f"Expected img embeds to be have 3 dimensions, got {num_dims}")
686 + num_images = 1 if num_dims == 2 else image_data.shape[0]
687 + else:
688 + # TODO - handle multiple image inputs once the API is solidified
689 + num_images = 1
690 +
691 + if prompt is None:
692 + prompt = tokenizer.decode(prompt_token_ids)
693 +
694 + # Drops anything between <img>/</img> tags; encoding with the tokenizer
695 + # will automatically add the image pads for the context.
696 + new_prompt, num_matched_images = re.subn(
697 + r"(Picture \d*: <img>).*?(<\/img>\n)",
698 + r"\1\2",
699 + prompt,
700 + )
701 +
702 + if num_matched_images != num_images:
703 + logger.warning(
704 + "Number of matched image placeholders %s doesn't match the number "
705 + "of expected images %s; check your placeholder formatting.",
706 + num_matched_images, num_images)
707 +
708 + new_prompt_token_ids = tokenizer.encode(new_prompt)
709 +
710 + return LLMInputs(prompt=new_prompt,
711 + prompt_token_ids=new_prompt_token_ids,
712 + multi_modal_data=multi_modal_data)
713 +
714 +
715 +def input_mapper_for_qwen(ctx: InputContext, data: object) -> MultiModalInputs:
716 + """Maps the input data to its MultiModalInputs (if any).
717 +
718 + Args:
719 + ctx: Context of the loaded model.
720 + data: data potentially containing image/image embeddings to be mapped
721 + to pixel_values in .forward() for a visual QWenLMHeadModel model.
722 +
723 + Returns:
724 + MultiModalInputs containing the stacked normalized images tensor or
725 + image embeddings.
726 + """
727 + # Early exit if we have provided an image to a language only Qwen model
728 + hf_config = ctx.get_hf_config()
729 + if not hasattr(hf_config, "visual"):
730 + logger.warning(
731 + "Images were provided but this model has no visual config; "
732 + "multimodal inputs will not be forwarded to the model.")
733 + return MultiModalInputs()
734 +
735 + model_config = ctx.model_config
736 + tokenizer = cached_get_tokenizer(model_config.tokenizer,
737 + trust_remote_code=True)
738 +
739 + image_pair_tok = tokenizer.encode(IMG_START + IMG_END,
740 + add_special_tokens=False,
741 + return_tensors="pt").squeeze()
742 + image_start_id = image_pair_tok[0]
743 + image_end_id = image_pair_tok[-1]
744 + if (image_start_id + 1) != image_end_id:
745 + raise ValueError(
746 + f"Found image end ID {image_end_id}, but expected {IMG_START} + 1")
747 + if len(image_pair_tok) != (MAX_QWEN_IMG_TOKENS + 2):
748 + raise ValueError(
749 + f"Expected image context length of {MAX_QWEN_IMG_TOKENS}, "
750 + f"but got {image_pair_tok - 2}")
751 +
752 + hf_config = ctx.get_hf_config()
753 + image_size = hf_config.visual["image_size"]
754 + img_emb_size = hf_config.visual["output_dim"]
755 +
756 + if isinstance(data, torch.Tensor):
757 + # It's expected that our values have already been processed
758 + # by the visual transformer; shape is expected to be:
759 + # (# images, 256, hidden_size)
760 + if len(data.shape) == 2:
761 + # Assume only one image embed was provided; unsqueeze the extra dim
762 + data = data.unsqueeze(0)
763 + if len(data.shape) != 3 or data.shape[
764 + 1] != MAX_QWEN_IMG_TOKENS or data.shape[2] != img_emb_size:
765 + raise ValueError(
766 + "Expected image embeds to be a tensor of shape"
767 + f"[# images, {MAX_QWEN_IMG_TOKENS}, {img_emb_size}], but "
768 + f"received shape [{data.shape}]")
769 + pixel_values = data
770 +
771 + else:
772 + transform = build_normalization_transform(image_size)
773 + # TODO - handle multiple image inputs once the API is solidified
774 + transformed_images = [transform(data)]
775 + pixel_values = torch.stack(transformed_images, dim=0)
776 + return MultiModalInputs({"pixel_values": pixel_values})
777 +
778 +
779 +def build_normalization_transform(image_size: int) -> transforms.Compose:
780 + """Builds a normalization transform which can be applied to one or
781 + more input images from which we want to extract visual features.
782 +
783 + Args:
784 + image_size: size of the image to be processed for visual embeddings.
785 +
786 + Returns:
787 + Callable transform for normalizing and resizing one RGB image.
788 + """
789 + return transforms.Compose([
790 + transforms.Resize((image_size, image_size),
791 + interpolation=InterpolationMode.BICUBIC),
792 + transforms.ToTensor(),
793 + transforms.Normalize(mean=CLIP_MEAN, std=CLIP_STD),
794 + ])
795 +
796 +
797 +def dummy_data_for_qwen(
798 + ctx: InputContext,
799 + seq_len: int,
800 + mm_counts: Mapping[str, int],
801 +) -> Tuple[SequenceData, Optional[Dict]]:
802 + """Build dummy data for warming up Qwen models; this will only contain text
803 + matching the defaults for VLLM unless the model has a visual config.
804 +
805 + Args:
806 + ctx: Context of the loaded model.
807 + seq_len: Number of tokens in the text sequence.
808 + mm_counts: multimodal data counts.
809 +
810 + Returns:
811 + Tuple containing sequential and multimodal data.
812 + """
813 + hf_config = ctx.get_hf_config()
814 +
815 + # The presence of a visual config indicates this is a multimodal model.
816 + # If we don't have it, the model is considered an LLM for warmup purposes.
817 + if not hasattr(hf_config, "visual"):
818 + seq_data = SequenceData(array(VLLM_TOKEN_ID_ARRAY_TYPE, [0] * seq_len))
819 + mm_data = None
820 + return seq_data, mm_data
821 +
822 + # We have a visual component - use images to warm up
823 + num_images = mm_counts["image"]
824 + model_config = ctx.model_config
825 + tokenizer = cached_get_tokenizer(model_config.tokenizer,
826 + trust_remote_code=True)
827 +
828 + # Build the image prompts with no imgpads; the tokenizer will add img pads
829 + image_prompt = ''.join(
830 + [get_image_text(idx, False) for idx in range(1, num_images + 1)])
831 + toks = tokenizer.encode(image_prompt, add_special_tokens=False)
832 +
833 + # Make sure we actually get the fixed context size per tok padding
834 + num_pads = toks.count(tokenizer.encode(IMG_PAD)[0])
835 + if num_pads != (num_images * MAX_QWEN_IMG_TOKENS):
836 + raise ValueError(
837 + f"Tokenized dummy data should encode {MAX_QWEN_IMG_TOKENS} pads"
838 + f" per image, but got {num_pads} pads for {num_images} image(s)"
839 + " in total. Are you using a qwen tokenizer?")
840 +
841 + # Ensure the number of tokens is at minimum the sequence length provided
842 + if len(toks) < seq_len:
843 + toks += [0] * (seq_len - len(toks))
844 +
845 + # Build the input images; width/height doesn't actually matter here since
846 + # the data will get resized and the # of tokens per image is constant
847 + image = Image.new("RGB", (224, 224), color=0)
848 + mm_data = {"image": image if num_images == 1 else [image] * num_images}
849 + return SequenceData(array(VLLM_TOKEN_ID_ARRAY_TYPE, toks)), mm_data
850 +
851 +
852 +@MULTIMODAL_REGISTRY.register_image_input_mapper(input_mapper_for_qwen)
853 +@MULTIMODAL_REGISTRY.register_max_image_tokens(MAX_QWEN_IMG_TOKENS)
854 +@INPUT_REGISTRY.register_dummy_data(dummy_data_for_qwen)
855 +@INPUT_REGISTRY.register_input_processor(input_processor_for_qwen)
856 +class QWenLMHeadModel(nn.Module, SupportsMultiModal):
857
858 def __init__(
859 self,
860 config: PretrainedConfig,
861 + multimodal_config: MultiModalConfig,
862 cache_config: Optional[CacheConfig] = None,
863 quant_config: Optional[QuantizationConfig] = None,
864 ):
865 super().__init__()
866 self.config = config
867 + self.multimodal_config = multimodal_config
868 self.quant_config = quant_config
869 self.transformer = QWenModel(config, cache_config, quant_config)
870 self.lm_head = ParallelLMHead(config.vocab_size,
@@ -257,16 +875,47 @@ class QWenLMHeadModel(nn.Module):
875 self.logits_processor = LogitsProcessor(config.vocab_size)
876 self.sampler = Sampler()
877
260 - def forward(
261 - self,
262 - input_ids: torch.Tensor,
263 - positions: torch.Tensor,
264 - kv_caches: List[torch.Tensor],
265 - attn_metadata: AttentionMetadata,
266 - intermediate_tensors: Optional[IntermediateTensors] = None,
267 - ) -> torch.Tensor:
878 + def _get_image_input_type(
879 + self,
880 + pixel_values: Optional[torch.Tensor]) -> Optional[QwenImageInputs]:
881 + """Determines if the provided pixel_values are normalized pixel values
882 + or image embeddings.
883 +
884 + Args:
885 + pixel_values: Optional data to processed into visual embeddings.
886 +
887 + Returns:
888 + None of the QwenImageInputs type used to determine whether or not
889 + the visual transformer needs to process the pixel_values.
890 + """
891 + if pixel_values is not None and self.transformer.visual is not None:
892 + pixel_values = flatten_bn(pixel_values)
893 + if len(pixel_values.shape) == 3 and pixel_values.shape[
894 + 1] == MAX_QWEN_IMG_TOKENS and pixel_values.shape[
895 + 2] == self.config.visual["output_dim"]:
896 + return QwenImageEmbeddingInputs(
897 + type="image_embeds",
898 + data=pixel_values,
899 + )
900 + else:
901 + # If we have the wrong shape, assume we still need to process
902 + return QwenImagePixelInputs(
903 + type="pixel_values",
904 + data=pixel_values,
905 + )
906 + return None
907 +
908 + def forward(self,
909 + input_ids: torch.Tensor,
910 + positions: torch.Tensor,
911 + kv_caches: List[torch.Tensor],
912 + attn_metadata: AttentionMetadata,
913 + intermediate_tensors: Optional[IntermediateTensors] = None,
914 + pixel_values: Optional[torch.Tensor] = None) -> torch.Tensor:
915 + pixel_values = self._get_image_input_type(pixel_values)
916 hidden_states = self.transformer(input_ids, positions, kv_caches,
269 - attn_metadata, intermediate_tensors)
917 + attn_metadata, intermediate_tensors,
918 + pixel_values)
919 return hidden_states
920
921 def make_empty_intermediate_tensors(
@@ -328,15 +977,6 @@ class QWenLMHeadModel(nn.Module):
977 # Skip loading extra bias for GPTQ models.
978 if name.endswith(".bias") and name not in params_dict:
979 continue
331 - # Skip loading visual weights to support Qwen-VL models
332 - # in cases with text-only inputs
333 - # TODO: add support for Qwen-VL
334 - if (name not in params_dict
335 - and name.startswith("transformer.visual.")):
336 - print_warning_once(
337 - "Only text inputs are allowed. Images won't be handled "
338 - "until Qwen-VL models are fully supported.")
339 - continue
980 # Skip layers on other devices.
981 if is_pp_missing_parameter(name, self):
982 continue