@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9e169a4c61
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[Model] Adding support for MiniCPM-V (#4087)

Alphi committed Jul 25, 2024 at 11:59 UTC 9e169a4c619c33ec4f9a14c5e971e3aa34bc4444
11 files changed +942 -18
docs/source/dev/multimodal/multimodal_index.rst
+2
@@ -40,6 +40,8 @@ Registry
40 Base Classes
41 ------------
42
43 +.. autodata:: vllm.multimodal.NestedTensors
44 +
45 .. autodata:: vllm.multimodal.BatchedTensors
46
47 .. autoclass:: vllm.multimodal.MultiModalDataBuiltins
docs/source/models/supported_models.rst
+4
@@ -206,6 +206,10 @@ Vision Language Models
206 - Phi-3-Vision
207 - :code:`microsoft/Phi-3-vision-128k-instruct`, etc.
208 -
209 + * - :code:`MiniCPM-V`
210 + - MiniCPM-V
211 + - :code:`openbmb/MiniCPM-V-2`, :code:`openbmb/MiniCPM-Llama3-V-2_5`, etc.
212 + -
213
214 If your model uses one of the above model architectures, you can seamlessly run your model with vLLM.
215 Otherwise, please refer to :ref:`Adding a New Model <adding_a_new_model>` and :ref:`Enabling Multimodal Inputs <enabling_multimodal_inputs>`
examples/minicpmv_example.py new
+53
@@ -0,0 +1,53 @@
1 +from transformers import AutoTokenizer
2 +
3 +from vllm import LLM, SamplingParams
4 +from vllm.assets.image import ImageAsset
5 +
6 +# 2.0
7 +# MODEL_NAME = "HwwwH/MiniCPM-V-2"
8 +# 2.5
9 +MODEL_NAME = "openbmb/MiniCPM-Llama3-V-2_5"
10 +
11 +image = ImageAsset("stop_sign").pil_image.convert("RGB")
12 +
13 +tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME, trust_remote_code=True)
14 +llm = LLM(model=MODEL_NAME,
15 + gpu_memory_utilization=1,
16 + trust_remote_code=True,
17 + max_model_len=4096)
18 +
19 +messages = [{
20 + 'role':
21 + 'user',
22 + 'content':
23 + '(<image>./</image>)\n' + "What's the content of the image?"
24 +}]
25 +prompt = tokenizer.apply_chat_template(messages,
26 + tokenize=False,
27 + add_generation_prompt=True)
28 +# 2.0
29 +# stop_token_ids = [tokenizer.eos_id]
30 +# 2.5
31 +stop_token_ids = [tokenizer.eos_id, tokenizer.eot_id]
32 +
33 +sampling_params = SamplingParams(
34 + stop_token_ids=stop_token_ids,
35 + # temperature=0.7,
36 + # top_p=0.8,
37 + # top_k=100,
38 + # seed=3472,
39 + max_tokens=1024,
40 + # min_tokens=150,
41 + temperature=0,
42 + use_beam_search=True,
43 + # length_penalty=1.2,
44 + best_of=3)
45 +
46 +outputs = llm.generate({
47 + "prompt": prompt,
48 + "multi_modal_data": {
49 + "image": image
50 + }
51 +},
52 + sampling_params=sampling_params)
53 +print(outputs[0].outputs[0].text)
tests/conftest.py
+6 -5
@@ -11,7 +11,7 @@ import torch.nn as nn
11 import torch.nn.functional as F
12 from PIL import Image
13 from transformers import (AutoModelForCausalLM, AutoModelForVision2Seq,
14 - AutoTokenizer, BatchEncoding)
14 + AutoTokenizer, BatchEncoding, BatchFeature)
15
16 from vllm import LLM, SamplingParams
17 from vllm.assets.image import ImageAsset
@@ -133,7 +133,7 @@ def image_assets() -> _ImageAssets:
133 return IMAGE_ASSETS
134
135
136 -_T = TypeVar("_T", nn.Module, torch.Tensor, BatchEncoding)
136 +_T = TypeVar("_T", nn.Module, torch.Tensor, BatchEncoding, BatchFeature)
137
138
139 class HfRunner:
@@ -339,7 +339,6 @@ class HfRunner:
339 processor_kwargs["images"] = images[i]
340
341 inputs = self.processor(**processor_kwargs)
342 - input_ids = inputs.input_ids
342
343 output = self.model.generate(
344 **self.wrap_device(inputs),
@@ -381,7 +380,7 @@ class HfRunner:
380
381 all_logprobs.append(seq_logprobs_lst)
382 seq_ids = output.sequences[0]
384 - output_len = seq_ids.shape[0] - input_ids.shape[1]
383 + output_len = len(seq_logprobs_lst)
384 output_ids = seq_ids[-output_len:]
385 all_output_ids.append(output_ids.tolist())
386 all_output_strs.append(self.tokenizer.decode(output_ids))
@@ -514,10 +513,12 @@ class VllmRunner:
513 max_tokens: int,
514 num_logprobs: int,
515 images: Optional[List[Image.Image]] = None,
516 + stop_token_ids: Optional[List[int]] = None,
517 ) -> List[Tuple[List[int], str, Optional[SampleLogprobs]]]:
518 greedy_logprobs_params = SamplingParams(temperature=0.0,
519 max_tokens=max_tokens,
520 - logprobs=num_logprobs)
520 + logprobs=num_logprobs,
521 + stop_token_ids=stop_token_ids)
522 outputs = self.generate_w_logprobs(prompts,
523 greedy_logprobs_params,
524 images=images)
tests/models/test_minicpmv.py new
+163
@@ -0,0 +1,163 @@
1 +from collections import UserDict
2 +from typing import List, Optional, Tuple, Type
3 +
4 +import pytest
5 +import torch
6 +import torch.types
7 +from transformers import BatchFeature
8 +
9 +from vllm.multimodal.utils import rescale_image_size
10 +from vllm.sequence import SampleLogprobs
11 +
12 +from ..conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
13 +from .utils import check_logprobs_close
14 +
15 +pytestmark = pytest.mark.vlm
16 +
17 +# The image token is placed before "user" on purpose so that the test can pass
18 +HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
19 + "stop_sign":
20 + "<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n" \
21 + "(<image>./</image>)\nWhat's the content of the image?<|eot_id|>" \
22 + "<|start_header_id|>assistant<|end_header_id|>\n\n", # noqa: E501
23 + "cherry_blossom":
24 + "<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n" \
25 + "(<image>./</image>)\nWhat is the season?<|eot_id|>" \
26 + "<|start_header_id|>assistant<|end_header_id|>\n\n"
27 +})
28 +
29 +models = ["openbmb/MiniCPM-Llama3-V-2_5"]
30 +
31 +
32 +def trunc_hf_output(hf_output: Tuple[List[int], str,
33 + Optional[SampleLogprobs]]):
34 + output_ids, output_str, out_logprobs = hf_output
35 + if output_str.endswith("<|eot_id|>"):
36 + output_str = output_str.split("<|eot_id|>")[0]
37 + return output_ids, output_str, out_logprobs
38 +
39 +
40 +target_dtype = "half"
41 +
42 +
43 +def run_test(
44 + hf_runner: Type[HfRunner],
45 + vllm_runner: Type[VllmRunner],
46 + image_assets: _ImageAssets,
47 + model: str,
48 + *,
49 + size_factors: List[float],
50 + dtype: str,
51 + max_tokens: int,
52 + num_logprobs: int,
53 + tensor_parallel_size: int,
54 + distributed_executor_backend: Optional[str] = None,
55 +):
56 + """Inference result should be the same between hf and vllm.
57 +
58 + All the image fixtures for the test is under tests/images.
59 + For huggingface runner, we provide the PIL images as input.
60 + For vllm runner, we provide MultiModalDataDict objects
61 + and corresponding vision language config as input.
62 + Note, the text input is also adjusted to abide by vllm contract.
63 + The text output is sanitized to be able to compare with hf.
64 + """
65 + images = [asset.pil_image for asset in image_assets]
66 +
67 + inputs_per_image = [(
68 + [prompt for _ in size_factors],
69 + [rescale_image_size(image, factor) for factor in size_factors],
70 + ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
71 +
72 + # NOTE: take care of the order. run vLLM first, and then run HF.
73 + # vLLM needs a fresh new process without cuda initialization.
74 + # if we run HF first, the cuda initialization will be done and it
75 + # will hurt multiprocessing backend with fork method (the default method).
76 +
77 + # max_model_len should be greater than image_feature_size
78 + with vllm_runner(model,
79 + max_model_len=4096,
80 + max_num_seqs=1,
81 + dtype=dtype,
82 + tensor_parallel_size=tensor_parallel_size,
83 + distributed_executor_backend=distributed_executor_backend,
84 + enforce_eager=True) as vllm_model:
85 + tokenizer = vllm_model.model.get_tokenizer()
86 + stop_token_ids = [tokenizer.eos_id, tokenizer.eot_id]
87 + vllm_outputs_per_image = [
88 + vllm_model.generate_greedy_logprobs(prompts,
89 + max_tokens,
90 + num_logprobs=num_logprobs,
91 + images=vllm_images,
92 + stop_token_ids=stop_token_ids)
93 + for prompts, vllm_images in inputs_per_image
94 + ]
95 +
96 + with hf_runner(model, dtype=dtype) as hf_model, torch.no_grad():
97 +
98 + class NestedInputs(UserDict):
99 +
100 + def __init__(self, model_inputs: BatchFeature):
101 + super().__init__({"model_inputs": model_inputs})
102 +
103 + self.model_inputs = model_inputs
104 +
105 + def to(self, device: torch.types.Device):
106 + return NestedInputs(self.model_inputs.to(device))
107 +
108 + hf_processor = hf_model.processor
109 + hf_model.processor = lambda **kw: NestedInputs(
110 + hf_processor(**kw) # type: ignore
111 + )
112 +
113 + hf_outputs_per_image = [
114 + hf_model.generate_greedy_logprobs_limit(prompts,
115 + max_tokens,
116 + num_logprobs=num_logprobs,
117 + images=hf_images,
118 + tokenizer=tokenizer)
119 + for prompts, hf_images in inputs_per_image
120 + ]
121 +
122 + for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
123 + vllm_outputs_per_image):
124 + check_logprobs_close(
125 + outputs_0_lst=[
126 + trunc_hf_output(hf_output) for hf_output in hf_outputs
127 + ],
128 + outputs_1_lst=vllm_outputs,
129 + name_0="hf",
130 + name_1="vllm",
131 + )
132 +
133 +
134 +@pytest.mark.parametrize("model", models)
135 +@pytest.mark.parametrize(
136 + "size_factors",
137 + [
138 + # No image
139 + [],
140 + # Single-scale
141 + [1.0],
142 + # Single-scale, batched
143 + [1.0, 1.0, 1.0],
144 + # Multi-scale
145 + [0.25, 0.5, 1.0],
146 + ],
147 +)
148 +@pytest.mark.parametrize("dtype", [target_dtype])
149 +@pytest.mark.parametrize("max_tokens", [128])
150 +@pytest.mark.parametrize("num_logprobs", [5])
151 +def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
152 + dtype: str, max_tokens: int, num_logprobs: int) -> None:
153 + run_test(
154 + hf_runner,
155 + vllm_runner,
156 + image_assets,
157 + model,
158 + size_factors=size_factors,
159 + dtype=dtype,
160 + max_tokens=max_tokens,
161 + num_logprobs=num_logprobs,
162 + tensor_parallel_size=1,
163 + )
vllm/model_executor/models/__init__.py
+1
@@ -50,6 +50,7 @@ _GENERATION_MODELS = {
50 "MptForCausalLM": ("mpt", "MPTForCausalLM"),
51 "MPTForCausalLM": ("mpt", "MPTForCausalLM"),
52 "MiniCPMForCausalLM": ("minicpm", "MiniCPMForCausalLM"),
53 + "MiniCPMV": ("minicpmv", "MiniCPMV"),
54 "OlmoForCausalLM": ("olmo", "OlmoForCausalLM"),
55 "OPTForCausalLM": ("opt", "OPTForCausalLM"),
56 "OrionForCausalLM": ("orion", "OrionForCausalLM"),
vllm/model_executor/models/llama.py
+3 -1
@@ -418,9 +418,11 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA):
418 kv_caches: List[torch.Tensor],
419 attn_metadata: AttentionMetadata,
420 intermediate_tensors: Optional[IntermediateTensors] = None,
421 + input_embeds: Optional[torch.Tensor] = None
422 ) -> Union[torch.Tensor, IntermediateTensors]:
423 model_output = self.model(input_ids, positions, kv_caches,
423 - attn_metadata, intermediate_tensors)
424 + attn_metadata, intermediate_tensors,
425 + input_embeds)
426 return model_output
427
428 def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/minicpm.py
+2 -1
@@ -463,10 +463,11 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA):
463 positions: torch.Tensor,
464 kv_caches: List[torch.Tensor],
465 attn_metadata: AttentionMetadata,
466 + input_embeds: Optional[torch.Tensor] = None,
467 intermediate_tensors: Optional[IntermediateTensors] = None,
468 ) -> torch.Tensor:
469 hidden_states = self.model(input_ids, positions, kv_caches,
469 - attn_metadata)
470 + attn_metadata, input_embeds)
471 return hidden_states
472
473 def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/minicpmv.py new
+682
@@ -0,0 +1,682 @@
1 +# coding=utf-8
2 +# Adapted from
3 +# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
4 +# Copyright 2023 The vLLM team.
5 +# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
6 +#
7 +# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
8 +# and OPT implementations in this library. It has been modified from its
9 +# original forms to accommodate minor architectural differences compared
10 +# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
11 +#
12 +# Licensed under the Apache License, Version 2.0 (the "License");
13 +# you may not use this file except in compliance with the License.
14 +# You may obtain a copy of the License at
15 +#
16 +# http://www.apache.org/licenses/LICENSE-2.0
17 +#
18 +# Unless required by applicable law or agreed to in writing, software
19 +# distributed under the License is distributed on an "AS IS" BASIS,
20 +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21 +# See the License for the specific language governing permissions and
22 +# limitations under the License.
23 +"""Inference-only MiniCPM-V-2 model compatible with HuggingFace weights."""
24 +import math
25 +import re
26 +from functools import partial
27 +from typing import Iterable, List, Optional, Tuple
28 +
29 +import numpy as np
30 +import torch
31 +import torch.nn.functional as F
32 +from PIL import Image
33 +from torch import nn
34 +from torch.nn.init import trunc_normal_
35 +from transformers.configuration_utils import PretrainedConfig
36 +from transformers.models.idefics2.modeling_idefics2 import (
37 + Idefics2VisionTransformer)
38 +
39 +from vllm.attention import AttentionMetadata
40 +from vllm.config import CacheConfig, MultiModalConfig
41 +from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
42 +from vllm.model_executor.layers.quantization.base_config import (
43 + QuantizationConfig)
44 +from vllm.model_executor.layers.sampler import Sampler
45 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
46 +from vllm.model_executor.models.interfaces import SupportsVision
47 +from vllm.model_executor.models.llama import LlamaForCausalLM
48 +from vllm.model_executor.models.minicpm import MiniCPMForCausalLM
49 +from vllm.model_executor.sampling_metadata import SamplingMetadata
50 +from vllm.multimodal import MULTIMODAL_REGISTRY
51 +from vllm.multimodal.image import (cached_get_image_processor,
52 + cached_get_tokenizer)
53 +from vllm.sequence import IntermediateTensors, SamplerOutput, SequenceData
54 +
55 +_KEYS_TO_MODIFY_MAPPING = {
56 + "language_model.lm_head": "lm_head",
57 + "language_model.model": "language_model",
58 +}
59 +
60 +
61 +def get_abs_pos(abs_pos, tgt_size):
62 + # abs_pos: L, C
63 + # tgt_size: (H, W)
64 + # return: M, C
65 + src_size = int(math.sqrt(abs_pos.size(0)))
66 + # tgt_size = int(math.sqrt(tgt_size))
67 + dtype = abs_pos.dtype
68 +
69 + return F.interpolate(
70 + abs_pos.float().reshape(1, src_size, src_size, -1).permute(0, 3, 1, 2),
71 + size=(tgt_size[0], tgt_size[1]),
72 + mode="bicubic",
73 + align_corners=False,
74 + ).permute(0, 2, 3, 1).flatten(0, 2).to(dtype=dtype)
75 +
76 +
77 +# https://github.com/facebookresearch/mae/blob/efb2a8062c206524e35e47d04501ed4f544c0ae8/util/pos_embed.py#L20
78 +def get_2d_sincos_pos_embed(embed_dim,
79 + grid_size,
80 + cls_token=False,
81 + version=2.0):
82 + """
83 + grid_size: int of the grid height and width
84 + return:
85 + pos_embed: [grid_size*grid_size, embed_dim] or
86 + [1+grid_size*grid_size, embed_dim] (w/ or w/o cls_token)
87 + """
88 + if isinstance(grid_size, int):
89 + grid_h_size, grid_w_size = grid_size, grid_size
90 + else:
91 + grid_h_size, grid_w_size = grid_size[0], grid_size[1]
92 +
93 + grid_h = np.arange(grid_h_size, dtype=np.float32)
94 + grid_w = np.arange(grid_w_size, dtype=np.float32)
95 + grid = np.meshgrid(grid_w, grid_h) # here w goes first
96 + grid = np.stack(grid, axis=0)
97 +
98 + if version == 2.0:
99 + grid = grid.reshape([2, 1, grid_h_size, grid_w_size])
100 + pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid, version)
101 + if cls_token:
102 + pos_embed = np.concatenate([np.zeros([1, embed_dim]), pos_embed],
103 + axis=0)
104 + else:
105 + pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid, version)
106 + return pos_embed
107 +
108 +
109 +def get_2d_sincos_pos_embed_from_grid(embed_dim, grid, version=2.0):
110 + assert embed_dim % 2 == 0
111 +
112 + # use half of dimensions to encode grid_h
113 + emb_h = get_1d_sincos_pos_embed_from_grid(
114 + embed_dim // 2, grid[0], version) # (H*W, D/2) or (H, W, D/2)
115 + emb_w = get_1d_sincos_pos_embed_from_grid(
116 + embed_dim // 2, grid[1], version) # (H*W, D/2) or (H, W, D/2)
117 +
118 + if version == 2.0:
119 + emb = np.concatenate([emb_h, emb_w], axis=1) # (H*W, D)
120 + else:
121 + emb = np.concatenate([emb_h, emb_w], axis=-1) # (H, W, D)
122 + return emb
123 +
124 +
125 +def get_1d_sincos_pos_embed_from_grid(embed_dim, pos, version=2.0):
126 + """
127 + embed_dim: output dimension for each position
128 + pos: a list of positions to be encoded: size (M,) / (H, W)
129 + out: (M, D) / (H, W, D)
130 + """
131 + assert embed_dim % 2 == 0
132 + omega = np.arange(embed_dim // 2, dtype=np.float32)
133 + omega /= embed_dim / 2.
134 + omega = 1. / 10000**omega # (D/2,)
135 +
136 + if version == 2.0:
137 + pos = pos.reshape(-1) # (M,)
138 + out = np.einsum('m,d->md', pos, omega) # (M, D/2), outer product
139 + emb_sin = np.sin(out) # (M, D/2)
140 + emb_cos = np.cos(out) # (M, D/2)
141 + emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D)
142 + else:
143 + out = np.einsum('hw,d->hwd', pos, omega) # (H, W, D/2), outer product
144 + emb_sin = np.sin(out) # (H, W, D/2)
145 + emb_cos = np.cos(out) # (H, W, D/2)
146 + emb = np.concatenate([emb_sin, emb_cos], axis=-1) # (H, W, D)
147 + return emb
148 +
149 +
150 +class Resampler(nn.Module):
151 + """
152 + A 2D perceiver-resampler network with one cross attention layers by
153 + (grid_size**2) learnable queries and 2d sincos pos_emb
154 + Outputs:
155 + A tensor with the shape of (grid_size**2, embed_dim)
156 + """
157 +
158 + default_norm_layer = partial(nn.LayerNorm, eps=1e-6)
159 +
160 + def __init__(self,
161 + num_queries,
162 + grid_size,
163 + embed_dim,
164 + num_heads,
165 + kv_dim=None,
166 + norm_layer=default_norm_layer,
167 + adaptive=False,
168 + max_size=(70, 70),
169 + version=2.0):
170 + super().__init__()
171 +
172 + self.version = version
173 + if self.version == 2.0:
174 + self.num_queries = grid_size**2
175 + else:
176 + self.num_queries = num_queries
177 + self.max_size = max_size
178 + self.embed_dim = embed_dim
179 + self.num_heads = num_heads
180 + self.adaptive = adaptive
181 +
182 + self.query = nn.Parameter(torch.zeros(self.num_queries, embed_dim))
183 + trunc_normal_(self.query, std=.02)
184 +
185 + if kv_dim is not None and kv_dim != embed_dim:
186 + self.kv_proj = nn.Linear(kv_dim, embed_dim, bias=False)
187 + else:
188 + self.kv_proj = nn.Identity()
189 +
190 + self.attn = nn.MultiheadAttention(embed_dim, num_heads)
191 + self.ln_q = norm_layer(embed_dim)
192 + self.ln_kv = norm_layer(embed_dim)
193 +
194 + self.ln_post = norm_layer(embed_dim)
195 + self.proj = nn.Parameter(
196 + (embed_dim**-0.5) * torch.randn(embed_dim, embed_dim))
197 +
198 + if self.version == 2.0:
199 + self.pos_embed = nn.Parameter(
200 + torch.from_numpy(
201 + get_2d_sincos_pos_embed(
202 + embed_dim, grid_size,
203 + version=self.version)).float()).requires_grad_(False)
204 + else:
205 + self._set_2d_pos_cache(self.max_size)
206 +
207 + self.apply(self._init_weights)
208 +
209 + def _set_2d_pos_cache(self, max_size, device='cpu'):
210 + pos_embed = torch.from_numpy(
211 + get_2d_sincos_pos_embed(self.embed_dim,
212 + max_size,
213 + version=self.version)).float().to(device)
214 + self.register_buffer("pos_embed", pos_embed, persistent=False)
215 +
216 + def _adjust_pos_cache(self, tgt_sizes, device):
217 + max_h = torch.max(tgt_sizes[:, 0])
218 + max_w = torch.max(tgt_sizes[:, 1])
219 + if max_h > self.max_size[0] or max_w > self.max_size[1]:
220 + self.max_size = [
221 + max(max_h, self.max_size[0]),
222 + max(max_w, self.max_size[1])
223 + ]
224 + self._set_2d_pos_cache(self.max_size, device)
225 +
226 + def _init_weights(self, m):
227 + if isinstance(m, nn.Linear):
228 + trunc_normal_(m.weight, std=.02)
229 + if isinstance(m, nn.Linear) and m.bias is not None:
230 + nn.init.constant_(m.bias, 0)
231 + elif isinstance(m, nn.LayerNorm):
232 + nn.init.constant_(m.bias, 0)
233 + nn.init.constant_(m.weight, 1.0)
234 +
235 + def forward_2_5(self, x, tgt_sizes=None):
236 + assert x.shape[0] == tgt_sizes.shape[0]
237 + bs = x.shape[0]
238 +
239 + device = x.device
240 + dtype = x.dtype
241 +
242 + patch_len = tgt_sizes[:, 0] * tgt_sizes[:, 1]
243 +
244 + self._adjust_pos_cache(tgt_sizes, device=device)
245 +
246 + max_patch_len = torch.max(patch_len)
247 + key_padding_mask = torch.zeros((bs, max_patch_len),
248 + dtype=torch.bool,
249 + device=device)
250 +
251 + pos_embed = []
252 + for i in range(bs):
253 + tgt_h, tgt_w = tgt_sizes[i]
254 + pos_embed.append(self.pos_embed[:tgt_h, :tgt_w, :].reshape(
255 + (tgt_h * tgt_w, -1)).to(dtype)) # patches * D
256 + key_padding_mask[i, patch_len[i]:] = True
257 +
258 + pos_embed = torch.nn.utils.rnn.pad_sequence(pos_embed,
259 + batch_first=True,
260 + padding_value=0.0).permute(
261 + 1, 0,
262 + 2) # BLD => L * B * D
263 +
264 + x = self.kv_proj(x) # B * L * D
265 + x = self.ln_kv(x).permute(1, 0, 2) # L * B * D
266 +
267 + q = self.ln_q(self.query) # Q * D
268 +
269 + out = self.attn(
270 + self._repeat(q, bs), # Q * B * D
271 + x + pos_embed, # L * B * D + L * B * D
272 + x,
273 + key_padding_mask=key_padding_mask)[0]
274 + # out: Q * B * D
275 + x = out.permute(1, 0, 2) # B * Q * D
276 +
277 + x = self.ln_post(x)
278 + x = x @ self.proj
279 + return x
280 +
281 + def forward_2(self, x, tgt_sizes=None, attn_mask=None):
282 + if self.adaptive:
283 + pos_embed = torch.Tensor(
284 + get_2d_sincos_pos_embed(self.embed_dim,
285 + tgt_sizes)).float().to(device=x.device,
286 + dtype=x.dtype)
287 + else:
288 + pos_embed = get_abs_pos(self.pos_embed, tgt_sizes)
289 +
290 + x = self.kv_proj(x)
291 + x = self.ln_kv(x).permute(1, 0, 2)
292 +
293 + N = x.shape[1]
294 + q = self.ln_q(self.query)
295 + out = self.attn(self._repeat(q, N) + self.pos_embed.unsqueeze(1),
296 + x + pos_embed.unsqueeze(1),
297 + x,
298 + attn_mask=attn_mask)[0]
299 + x = out.permute(1, 0, 2)
300 +
301 + x = self.ln_post(x)
302 + x = x @ self.proj
303 + return x
304 +
305 + def forward(self, x, tgt_sizes=None, attn_mask=None):
306 + if self.version == 2.0:
307 + return self.forward_2(x, tgt_sizes=tgt_sizes, attn_mask=attn_mask)
308 + else:
309 + return self.forward_2_5(x, tgt_sizes=tgt_sizes)
310 +
311 + def _repeat(self, query, N: int):
312 + return query.unsqueeze(1).repeat(1, N, 1)
313 +
314 +
315 +def get_max_minicpmv_image_tokens(ctx: InputContext):
316 + hf_config = ctx.get_hf_config(PretrainedConfig)
317 + return getattr(hf_config, "query_num", 64)
318 +
319 +
320 +def dummy_seq_data_for_minicpmv(seq_len: int):
321 + token_ids = [0] * seq_len
322 + return SequenceData(token_ids)
323 +
324 +
325 +def dummy_image_for_minicpmv(hf_config):
326 + width = height = hf_config.image_size
327 + image = Image.new("RGB", (width, height), color=0)
328 + return {"image": image}
329 +
330 +
331 +def dummy_data_for_minicpmv(ctx: InputContext, seq_len: int):
332 + hf_config = ctx.get_hf_config(PretrainedConfig)
333 +
334 + # image_feature_size = get_max_minicpmv_image_tokens(ctx)
335 +
336 + seq_data = dummy_seq_data_for_minicpmv(seq_len)
337 +
338 + mm_data = dummy_image_for_minicpmv(hf_config)
339 +
340 + return seq_data, mm_data
341 +
342 +
343 +def input_processor_for_minicpmv(ctx: InputContext, llm_inputs: LLMInputs):
344 + multi_modal_data = llm_inputs.get("multi_modal_data")
345 + if multi_modal_data is None or "image" not in multi_modal_data:
346 + return llm_inputs
347 +
348 + model_config = ctx.model_config
349 +
350 + tokenizer = cached_get_tokenizer(model_config.tokenizer,
351 + trust_remote_code=True)
352 +
353 + prompt = llm_inputs.get("prompt")
354 + if prompt is None:
355 + token_ids = llm_inputs.get("prompt_token_ids")
356 + prompt = tokenizer.decode(token_ids)
357 + image_processor = cached_get_image_processor(model_config.tokenizer)
358 +
359 + pattern = "(<image>./</image>)"
360 + image = multi_modal_data["image"]
361 + image_tags = re.findall(pattern, prompt)
362 + assert len(image_tags) <= 1
363 + text_chunks = prompt.split(pattern)
364 + new_prompt = text_chunks[0] \
365 + + image_processor.get_slice_image_placeholder(image.size) \
366 + + text_chunks[1]
367 +
368 + new_token_ids = tokenizer.encode(new_prompt)
369 +
370 + llm_inputs = LLMInputs(prompt_token_ids=new_token_ids,
371 + prompt=new_prompt,
372 + multi_modal_data=multi_modal_data)
373 + return llm_inputs
374 +
375 +
376 +@MULTIMODAL_REGISTRY.register_image_input_mapper()
377 +@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_minicpmv_image_tokens)
378 +@INPUT_REGISTRY.register_dummy_data(dummy_data_for_minicpmv)
379 +@INPUT_REGISTRY.register_input_processor(input_processor_for_minicpmv)
380 +class MiniCPMV(nn.Module, SupportsVision):
381 +
382 + def __init__(
383 + self,
384 + config,
385 + multimodal_config: MultiModalConfig,
386 + cache_config: Optional[CacheConfig] = None,
387 + quant_config: Optional[QuantizationConfig] = None,
388 + ):
389 + super().__init__()
390 + self.config = config
391 + self.multimodal_config = multimodal_config
392 +
393 + self.version = float(self.config.version)
394 + self.llm = self.init_llm(config, cache_config, quant_config)
395 + self.vpm = self.init_vision_module()
396 + param_dtype = torch.get_default_dtype()
397 + self.vpm.to(dtype=param_dtype)
398 + self.vision_dim = self.vpm.embed_dim if self.version == 2.0 \
399 + else self.vpm.embeddings.embed_dim
400 + self.embed_dim = self.llm.config.hidden_size
401 + self.resampler = self.init_resampler(self.embed_dim, self.vision_dim)
402 + self.resampler.to(device="cuda", dtype=param_dtype)
403 + self.sampler = Sampler()
404 +
405 + def init_llm(self, config, cache_config, quant_config):
406 + if self.version == 2.0:
407 + return MiniCPMForCausalLM(config,
408 + cache_config=cache_config,
409 + quant_config=quant_config)
410 + else:
411 + return LlamaForCausalLM(config,
412 + cache_config=cache_config,
413 + quant_config=quant_config)
414 +
415 + def init_vision_module(self):
416 + if self.version == 2.0:
417 + try:
418 + import timm
419 + except ImportError:
420 + raise ImportError(
421 + 'Please install timm==0.9.10') from ImportError
422 + default_dtype = torch.get_default_dtype()
423 + torch.set_default_dtype(torch.float16)
424 + model = timm.create_model('vit_so400m_patch14_siglip_384.webli',
425 + pretrained=False,
426 + num_classes=0,
427 + dynamic_img_size=True,
428 + dynamic_img_pad=True)
429 + torch.set_default_dtype(default_dtype)
430 + if isinstance(model, timm.models.VisionTransformer
431 + ) and model.attn_pool is not None:
432 + model.attn_pool = torch.nn.Identity()
433 +
434 + if self.config.drop_vision_last_layer:
435 + model.blocks = model.blocks[:-1]
436 + else:
437 + model = Idefics2VisionTransformer(self.config.vision_config)
438 + if self.config.drop_vision_last_layer:
439 + model.encoder.layers = model.encoder.layers[:-1]
440 + return model
441 +
442 + def init_resampler(self, embed_dim, vision_dim):
443 + default_dtype = torch.get_default_dtype()
444 + torch.set_default_dtype(torch.float16)
445 + if self.version == 2.0:
446 + resampler = Resampler(grid_size=int(
447 + math.sqrt(self.config.query_num)),
448 + num_queries=None,
449 + embed_dim=embed_dim,
450 + num_heads=embed_dim // 128,
451 + kv_dim=vision_dim,
452 + adaptive=True,
453 + version=self.version)
454 + else:
455 + resampler = Resampler(num_queries=self.config.query_num,
456 + grid_size=None,
457 + embed_dim=embed_dim,
458 + num_heads=embed_dim // 128,
459 + kv_dim=vision_dim,
460 + adaptive=True,
461 + version=self.version)
462 + torch.set_default_dtype(default_dtype)
463 + return resampler
464 +
465 + def get_vision_embedding(self,
466 + pixel_values,
467 + patch_attn_mask=None,
468 + tgt_sizes=None,
469 + version=2.0):
470 + if version == 2.0:
471 + res = []
472 + dtype = self.vpm.pos_embed.data.dtype
473 + for pixel_value in pixel_values:
474 + # V2.0 start
475 + H, W = pixel_value[0].shape[-2:]
476 + tgt_size = (math.ceil(H / self.vpm.patch_embed.patch_size[0]),
477 + math.ceil(W / self.vpm.patch_embed.patch_size[0]))
478 + # V2.0 end
479 + vision_embedding = self.vpm.forward_features(
480 + pixel_value.unsqueeze(0).type(dtype))
481 + if hasattr(self.vpm, 'num_prefix_tokens'
482 + ) and self.vpm.num_prefix_tokens > 0:
483 + vision_embedding = vision_embedding[:, self.vpm.
484 + num_prefix_tokens:]
485 + res.append(self.resampler(vision_embedding, tgt_size))
486 + return torch.vstack(res)
487 + else:
488 + vision_embedding = self.vpm(
489 + pixel_values.type(dtype),
490 + patch_attention_mask=patch_attn_mask).last_hidden_state
491 + vision_embedding = self.resampler(vision_embedding, tgt_sizes)
492 +
493 + def get_image_bounds(self, input_ids):
494 + tokenizer = cached_get_tokenizer(self.config._name_or_path,
495 + trust_remote_code=True)
496 + im_start_token_id = tokenizer.im_start_id
497 + im_end_token_id = tokenizer.im_end_id
498 + image_start_tokens = torch.where(input_ids == im_start_token_id)[0]
499 + image_start_tokens += 1
500 + image_end_tokens = torch.where(input_ids == im_end_token_id)[0]
501 + valid_image_nums = min(len(image_start_tokens), len(image_end_tokens))
502 + if valid_image_nums == 0:
503 + return []
504 + image_bound = torch.hstack([
505 + image_start_tokens[:valid_image_nums].unsqueeze(-1),
506 + image_end_tokens[:valid_image_nums].unsqueeze(-1),
507 + ])
508 +
509 + return image_bound
510 +
511 + def get_vision_hidden_states(self, data):
512 + if "vision_hidden_states" not in data:
513 + pixel_values = data["pixel_values"]
514 + tgt_sizes = data["tgt_sizes"]
515 + vision_hidden_states = []
516 + if self.version == 2.0:
517 + if pixel_values is not None and len(pixel_values) > 0:
518 + vision_hidden_states = self.get_vision_embedding(
519 + pixel_values)
520 + else:
521 + vision_hidden_states = torch.tensor([]).to(
522 + data["input_ids"].device)
523 + else:
524 + device = self.vpm.embeddings.position_embedding.weight.device
525 + dtype = self.vpm.embeddings.position_embedding.weight.dtype
526 + all_pixel_values = [
527 + i.flatten(end_dim=1).permute(1, 0) for i in pixel_values
528 + ]
529 + if all_pixel_values:
530 + tgt_sizes = torch.vstack(tgt_sizes).type(torch.int32)
531 + max_patches = torch.max(tgt_sizes[:, 0] * tgt_sizes[:, 1])
532 + all_pixel_values = torch.nn.utils.rnn.pad_sequence(
533 + all_pixel_values, batch_first=True, padding_value=0.0)
534 + B, L, _ = all_pixel_values.shape
535 + all_pixel_values = all_pixel_values.permute(
536 + 0, 2, 1).reshape(B, 3, -1, L)
537 +
538 + patch_attn_mask = torch.zeros((B, 1, max_patches),
539 + dtype=torch.bool,
540 + device=device)
541 + for i in range(B):
542 + patch_attn_mask[i, :tgt_sizes[i][0] *
543 + tgt_sizes[i][1]] = True
544 +
545 + vision_embedding = self.vpm(
546 + all_pixel_values.type(dtype),
547 + patch_attention_mask=patch_attn_mask).last_hidden_state
548 + vision_hidden_states = self.resampler(
549 + vision_embedding, tgt_sizes)
550 +
551 + else: # no image
552 + dummy_feature = []
553 + vision_hidden_states = dummy_feature
554 + else:
555 + vision_hidden_states = data["vision_hidden_states"]
556 +
557 + return vision_hidden_states
558 +
559 + def get_embedding(self, data):
560 + input_ids = data["input_ids"]
561 +
562 + vision_hidden_states = self.get_vision_hidden_states(data)
563 + if vision_hidden_states is not None and len(vision_hidden_states) > 0:
564 + image_bounds = self.get_image_bounds(input_ids)
565 + else:
566 + image_bounds = []
567 +
568 + if hasattr(self.llm.config, 'scale_emb'):
569 + vlm_embedding = self.llm.model.embed_tokens(
570 + input_ids) * self.llm.config.scale_emb
571 + else:
572 + vlm_embedding = self.llm.model.embed_tokens(input_ids)
573 + vision_hidden_states = [
574 + i.type(vlm_embedding.dtype) if isinstance(i, torch.Tensor) else i
575 + for i in vision_hidden_states
576 + ]
577 +
578 + if len(vision_hidden_states) > 0 and len(image_bounds) > 0:
579 + vision_hidden_states = torch.cat(vision_hidden_states, dim=0)
580 + image_indices = torch.stack([
581 + torch.arange(r[0], r[1], dtype=torch.long)
582 + for r in image_bounds
583 + ]).to(vlm_embedding.device)
584 + vlm_embedding.scatter_(
585 + 0,
586 + image_indices.view(-1, 1).repeat(1, vlm_embedding.shape[-1]),
587 + vision_hidden_states.view(-1, vision_hidden_states.shape[-1]))
588 + return vlm_embedding, vision_hidden_states
589 +
590 + def process_multimodal_inputs(self, inputs):
591 + pixel_values = []
592 + tgt_sizes = []
593 + for b in range(len(inputs["pixel_values"])):
594 + pixel_values += inputs["pixel_values"][b]
595 + tgt_sizes += inputs["tgt_sizes"][b]
596 + return {
597 + "pixel_values": pixel_values,
598 + "input_ids": inputs["input_ids"],
599 + "tgt_sizes": tgt_sizes
600 + }
601 +
602 + def forward(
603 + self,
604 + input_ids: torch.Tensor,
605 + positions: torch.Tensor,
606 + kv_caches: List[torch.Tensor],
607 + attn_metadata: AttentionMetadata,
608 + intermediate_tensors: Optional[IntermediateTensors] = None,
609 + **kwargs: object,
610 + ):
611 + inputs = {
612 + "pixel_values": kwargs.pop("pixel_values", []),
613 + "input_ids": input_ids,
614 + "tgt_sizes": kwargs.pop("tgt_sizes", None),
615 + }
616 +
617 + inputs = self.process_multimodal_inputs(inputs)
618 +
619 + vlm_embeddings, vision_hidden_states = self.get_embedding(inputs)
620 +
621 + output = self.llm(input_ids=None,
622 + positions=positions,
623 + kv_caches=kv_caches,
624 + attn_metadata=attn_metadata,
625 + intermediate_tensors=intermediate_tensors,
626 + input_embeds=vlm_embeddings)
627 + return output
628 +
629 + def compute_logits(self, hidden_states: torch.Tensor,
630 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
631 + return self.llm.compute_logits(hidden_states, sampling_metadata)
632 +
633 + def sample(
634 + self,
635 + logits: torch.Tensor,
636 + sampling_metadata: SamplingMetadata,
637 + ) -> Optional[SamplerOutput]:
638 + next_tokens = self.llm.sample(logits, sampling_metadata)
639 + return next_tokens
640 +
641 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
642 + stacked_params_mapping = [
643 + # (param_name, shard_name, shard_id)
644 + ("qkv_proj", "q_proj", "q"),
645 + ("qkv_proj", "k_proj", "k"),
646 + ("qkv_proj", "v_proj", "v"),
647 + ("gate_up_proj", "gate_proj", 0),
648 + ("gate_up_proj", "up_proj", 1),
649 + ]
650 + params_dict = dict(self.named_parameters())
651 + for name, loaded_weight in weights:
652 + # for key_to_modify, new_key in _KEYS_TO_MODIFY_MAPPING.items():
653 + # if key_to_modify in name:
654 + # name = name.replace(key_to_modify, new_key)
655 + if "rotary_emb.inv_freq" in name:
656 + continue
657 + if ("rotary_emb.cos_cached" in name
658 + or "rotary_emb.sin_cached" in name):
659 + # Models trained using ColossalAI may include these tensors in
660 + # the checkpoint. Skip them.
661 + continue
662 + use_default_weight_loading = False
663 + if "vpm" in name or 'resampler' in name:
664 + # We only do sharding for language model and
665 + # not vision model for now.
666 + use_default_weight_loading = True
667 + else:
668 + for (param_name, weight_name,
669 + shard_id) in stacked_params_mapping:
670 + if weight_name not in name:
671 + continue
672 + param = params_dict[name.replace(weight_name, param_name)]
673 + weight_loader = param.weight_loader
674 + weight_loader(param, loaded_weight, shard_id)
675 + break
676 + else:
677 + use_default_weight_loading = True
678 + if use_default_weight_loading:
679 + param = params_dict[name]
680 + weight_loader = getattr(param, "weight_loader",
681 + default_weight_loader)
682 + weight_loader(param, loaded_weight)
vllm/multimodal/__init__.py
+2 -1
@@ -1,5 +1,5 @@
1 from .base import (BatchedTensors, MultiModalDataBuiltins, MultiModalDataDict,
2 - MultiModalInputs, MultiModalPlugin)
2 + MultiModalInputs, MultiModalPlugin, NestedTensors)
3 from .registry import MultiModalRegistry
4
5 MULTIMODAL_REGISTRY = MultiModalRegistry()
@@ -17,6 +17,7 @@ __all__ = [
17 "MultiModalDataDict",
18 "MultiModalInputs",
19 "MultiModalPlugin",
20 + "NestedTensors",
21 "MULTIMODAL_REGISTRY",
22 "MultiModalRegistry",
23 ]
vllm/multimodal/base.py
+24 -10
@@ -2,7 +2,7 @@ import sys
2 from abc import ABC, abstractmethod
3 from collections import UserDict, defaultdict
4 from typing import (Any, Callable, Dict, List, Optional, Type, TypedDict,
5 - TypeVar, Union)
5 + TypeVar, Union, cast)
6
7 import torch
8 import torch.types
@@ -15,10 +15,17 @@ from vllm.logger import init_logger
15
16 logger = init_logger(__name__)
17
18 -BatchedTensors = Union[torch.Tensor, List[torch.Tensor]]
18 +NestedTensors = Union[List[torch.Tensor], torch.Tensor]
19 +"""
20 +Use a list instead of a tensor if the dimensions of each element do not match.
21 +Currently only supports up to singly nested list of tensors.
22 +"""
23 +
24 +BatchedTensors = Union[List[NestedTensors], NestedTensors]
25 """
26 If each input tensor in the batch has the same size, this is a single batched
21 -tensor; otherwise, this is a list of tensors with one element per batch.
27 +tensor; otherwise, this is a list of :class:`NestedTensors` with one element
28 +per item in the batch.
29 """
30
31 if sys.version_info < (3, 9):
@@ -27,7 +34,7 @@ if sys.version_info < (3, 9):
34 pass
35 else:
36
30 - class _MultiModalInputsBase(UserDict[str, torch.Tensor]):
37 + class _MultiModalInputsBase(UserDict[str, NestedTensors]):
38 pass
39
40
@@ -39,19 +46,26 @@ class MultiModalInputs(_MultiModalInputsBase):
46
47 @staticmethod
48 def try_concat(
42 - tensors: List[torch.Tensor],
49 + tensors: List[NestedTensors],
50 *,
51 device: torch.types.Device,
52 ) -> BatchedTensors:
46 - unbatched_shape = tensors[0].shape[1:]
53 + # may be list rather than tensors
54 + if isinstance(tensors[0], list):
55 + return [[t.to(device=device) for t in tensor[0]]
56 + for tensor in tensors]
57 +
58 + tensors_ = cast(List[torch.Tensor], tensors)
59 +
60 + unbatched_shape = tensors_[0].shape[1:]
61
48 - for tensor in tensors:
62 + for tensor in tensors_:
63 if tensor.shape[1:] != unbatched_shape:
64 return [
51 - tensor.squeeze(0).to(device=device) for tensor in tensors
65 + tensor.squeeze(0).to(device=device) for tensor in tensors_
66 ]
67
54 - return torch.cat(tensors, dim=0).to(device=device)
68 + return torch.cat(tensors_, dim=0).to(device=device)
69
70 @staticmethod
71 def batch(
@@ -64,7 +78,7 @@ class MultiModalInputs(_MultiModalInputsBase):
78
79 keys = inputs_list[0].keys()
80
67 - item_lists: Dict[str, List[torch.Tensor]] = defaultdict(list)
81 + item_lists: Dict[str, List[NestedTensors]] = defaultdict(list)
82
83 for inputs in inputs_list:
84 if inputs.keys() != keys: