A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[Model] Adding support for MiniCPM-V (#4087)
Alphi committed
Jul 25, 2024 at 11:59 UTC
9e169a4c619c33ec4f9a14c5e971e3aa34bc4444
11 files changed
+942
-18
docs/source/dev/multimodal/multimodal_index.rst
+2
@@ -40,6 +40,8 @@ Registry
40
Base Classes
41
------------
42
43
+.. autodata:: vllm.multimodal.NestedTensors
44
+
45
.. autodata:: vllm.multimodal.BatchedTensors
46
47
.. autoclass:: vllm.multimodal.MultiModalDataBuiltins
docs/source/models/supported_models.rst
+4
@@ -206,6 +206,10 @@ Vision Language Models
206
- Phi-3-Vision
207
- :code:`microsoft/Phi-3-vision-128k-instruct`, etc.
208
-
209
+ * - :code:`MiniCPM-V`
210
+ - MiniCPM-V
211
+ - :code:`openbmb/MiniCPM-V-2`, :code:`openbmb/MiniCPM-Llama3-V-2_5`, etc.
212
+ -
213
214
If your model uses one of the above model architectures, you can seamlessly run your model with vLLM.
215
Otherwise, please refer to :ref:`Adding a New Model <adding_a_new_model>` and :ref:`Enabling Multimodal Inputs <enabling_multimodal_inputs>`
examples/minicpmv_example.py
new
+53
@@ -0,0 +1,53 @@
1
+from transformers import AutoTokenizer
2
+
3
+from vllm import LLM, SamplingParams
4
+from vllm.assets.image import ImageAsset
5
+
6
+# 2.0
7
+# MODEL_NAME = "HwwwH/MiniCPM-V-2"
8
+# 2.5
9
+MODEL_NAME = "openbmb/MiniCPM-Llama3-V-2_5"
10
+
11
+image = ImageAsset("stop_sign").pil_image.convert("RGB")
12
+
13
+tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME, trust_remote_code=True)
14
+llm = LLM(model=MODEL_NAME,
15
+ gpu_memory_utilization=1,
16
+ trust_remote_code=True,
17
+ max_model_len=4096)
18
+
19
+messages = [{
20
+ 'role':
21
+ 'user',
22
+ 'content':
23
+ '(<image>./</image>)\n' + "What's the content of the image?"
24
+}]
25
+prompt = tokenizer.apply_chat_template(messages,
26
+ tokenize=False,
27
+ add_generation_prompt=True)
28
+# 2.0
29
+# stop_token_ids = [tokenizer.eos_id]
30
+# 2.5
31
+stop_token_ids = [tokenizer.eos_id, tokenizer.eot_id]
32
+
33
+sampling_params = SamplingParams(
34
+ stop_token_ids=stop_token_ids,
35
+ # temperature=0.7,
36
+ # top_p=0.8,
37
+ # top_k=100,
38
+ # seed=3472,
39
+ max_tokens=1024,
40
+ # min_tokens=150,
41
+ temperature=0,
42
+ use_beam_search=True,
43
+ # length_penalty=1.2,
44
+ best_of=3)
45
+
46
+outputs = llm.generate({
47
+ "prompt": prompt,
48
+ "multi_modal_data": {
49
+ "image": image
50
+ }
51
+},
52
+ sampling_params=sampling_params)
53
+print(outputs[0].outputs[0].text)
tests/conftest.py
+6
-5
@@ -11,7 +11,7 @@ import torch.nn as nn
11
import torch.nn.functional as F
12
from PIL import Image
13
from transformers import (AutoModelForCausalLM, AutoModelForVision2Seq,
14
- AutoTokenizer, BatchEncoding)
14
+ AutoTokenizer, BatchEncoding, BatchFeature)
15
16
from vllm import LLM, SamplingParams
17
from vllm.assets.image import ImageAsset
@@ -133,7 +133,7 @@ def image_assets() -> _ImageAssets:
133
return IMAGE_ASSETS
134
135
136
-_T = TypeVar("_T", nn.Module, torch.Tensor, BatchEncoding)
136
+_T = TypeVar("_T", nn.Module, torch.Tensor, BatchEncoding, BatchFeature)
137
138
139
class HfRunner:
@@ -339,7 +339,6 @@ class HfRunner:
339
processor_kwargs["images"] = images[i]
340
341
inputs = self.processor(**processor_kwargs)
342
- input_ids = inputs.input_ids
342
343
output = self.model.generate(
344
**self.wrap_device(inputs),
@@ -381,7 +380,7 @@ class HfRunner:
380
381
all_logprobs.append(seq_logprobs_lst)
382
seq_ids = output.sequences[0]
384
- output_len = seq_ids.shape[0] - input_ids.shape[1]
383
+ output_len = len(seq_logprobs_lst)
384
output_ids = seq_ids[-output_len:]
385
all_output_ids.append(output_ids.tolist())
386
all_output_strs.append(self.tokenizer.decode(output_ids))
@@ -514,10 +513,12 @@ class VllmRunner:
513
max_tokens: int,
514
num_logprobs: int,
515
images: Optional[List[Image.Image]] = None,
516
+ stop_token_ids: Optional[List[int]] = None,
517
) -> List[Tuple[List[int], str, Optional[SampleLogprobs]]]:
518
greedy_logprobs_params = SamplingParams(temperature=0.0,
519
max_tokens=max_tokens,
520
- logprobs=num_logprobs)
520
+ logprobs=num_logprobs,
521
+ stop_token_ids=stop_token_ids)
522
outputs = self.generate_w_logprobs(prompts,
523
greedy_logprobs_params,
524
images=images)
tests/models/test_minicpmv.py
new
+163
@@ -0,0 +1,163 @@
1
+from collections import UserDict
2
+from typing import List, Optional, Tuple, Type
3
+
4
+import pytest
5
+import torch
6
+import torch.types
7
+from transformers import BatchFeature
8
+
9
+from vllm.multimodal.utils import rescale_image_size
10
+from vllm.sequence import SampleLogprobs
11
+
12
+from ..conftest import IMAGE_ASSETS, HfRunner, VllmRunner, _ImageAssets
13
+from .utils import check_logprobs_close
14
+
15
+pytestmark = pytest.mark.vlm
16
+
17
+# The image token is placed before "user" on purpose so that the test can pass
18
+HF_IMAGE_PROMPTS = IMAGE_ASSETS.prompts({
19
+ "stop_sign":
20
+ "<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n" \
21
+ "(<image>./</image>)\nWhat's the content of the image?<|eot_id|>" \
22
+ "<|start_header_id|>assistant<|end_header_id|>\n\n", # noqa: E501
23
+ "cherry_blossom":
24
+ "<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n" \
25
+ "(<image>./</image>)\nWhat is the season?<|eot_id|>" \
26
+ "<|start_header_id|>assistant<|end_header_id|>\n\n"
27
+})
28
+
29
+models = ["openbmb/MiniCPM-Llama3-V-2_5"]
30
+
31
+
32
+def trunc_hf_output(hf_output: Tuple[List[int], str,
33
+ Optional[SampleLogprobs]]):
34
+ output_ids, output_str, out_logprobs = hf_output
35
+ if output_str.endswith("<|eot_id|>"):
36
+ output_str = output_str.split("<|eot_id|>")[0]
37
+ return output_ids, output_str, out_logprobs
38
+
39
+
40
+target_dtype = "half"
41
+
42
+
43
+def run_test(
44
+ hf_runner: Type[HfRunner],
45
+ vllm_runner: Type[VllmRunner],
46
+ image_assets: _ImageAssets,
47
+ model: str,
48
+ *,
49
+ size_factors: List[float],
50
+ dtype: str,
51
+ max_tokens: int,
52
+ num_logprobs: int,
53
+ tensor_parallel_size: int,
54
+ distributed_executor_backend: Optional[str] = None,
55
+):
56
+ """Inference result should be the same between hf and vllm.
57
+
58
+ All the image fixtures for the test is under tests/images.
59
+ For huggingface runner, we provide the PIL images as input.
60
+ For vllm runner, we provide MultiModalDataDict objects
61
+ and corresponding vision language config as input.
62
+ Note, the text input is also adjusted to abide by vllm contract.
63
+ The text output is sanitized to be able to compare with hf.
64
+ """
65
+ images = [asset.pil_image for asset in image_assets]
66
+
67
+ inputs_per_image = [(
68
+ [prompt for _ in size_factors],
69
+ [rescale_image_size(image, factor) for factor in size_factors],
70
+ ) for image, prompt in zip(images, HF_IMAGE_PROMPTS)]
71
+
72
+ # NOTE: take care of the order. run vLLM first, and then run HF.
73
+ # vLLM needs a fresh new process without cuda initialization.
74
+ # if we run HF first, the cuda initialization will be done and it
75
+ # will hurt multiprocessing backend with fork method (the default method).
76
+
77
+ # max_model_len should be greater than image_feature_size
78
+ with vllm_runner(model,
79
+ max_model_len=4096,
80
+ max_num_seqs=1,
81
+ dtype=dtype,
82
+ tensor_parallel_size=tensor_parallel_size,
83
+ distributed_executor_backend=distributed_executor_backend,
84
+ enforce_eager=True) as vllm_model:
85
+ tokenizer = vllm_model.model.get_tokenizer()
86
+ stop_token_ids = [tokenizer.eos_id, tokenizer.eot_id]
87
+ vllm_outputs_per_image = [
88
+ vllm_model.generate_greedy_logprobs(prompts,
89
+ max_tokens,
90
+ num_logprobs=num_logprobs,
91
+ images=vllm_images,
92
+ stop_token_ids=stop_token_ids)
93
+ for prompts, vllm_images in inputs_per_image
94
+ ]
95
+
96
+ with hf_runner(model, dtype=dtype) as hf_model, torch.no_grad():
97
+
98
+ class NestedInputs(UserDict):
99
+
100
+ def __init__(self, model_inputs: BatchFeature):
101
+ super().__init__({"model_inputs": model_inputs})
102
+
103
+ self.model_inputs = model_inputs
104
+
105
+ def to(self, device: torch.types.Device):
106
+ return NestedInputs(self.model_inputs.to(device))
107
+
108
+ hf_processor = hf_model.processor
109
+ hf_model.processor = lambda **kw: NestedInputs(
110
+ hf_processor(**kw) # type: ignore
111
+ )
112
+
113
+ hf_outputs_per_image = [
114
+ hf_model.generate_greedy_logprobs_limit(prompts,
115
+ max_tokens,
116
+ num_logprobs=num_logprobs,
117
+ images=hf_images,
118
+ tokenizer=tokenizer)
119
+ for prompts, hf_images in inputs_per_image
120
+ ]
121
+
122
+ for hf_outputs, vllm_outputs in zip(hf_outputs_per_image,
123
+ vllm_outputs_per_image):
124
+ check_logprobs_close(
125
+ outputs_0_lst=[
126
+ trunc_hf_output(hf_output) for hf_output in hf_outputs
127
+ ],
128
+ outputs_1_lst=vllm_outputs,
129
+ name_0="hf",
130
+ name_1="vllm",
131
+ )
132
+
133
+
134
+@pytest.mark.parametrize("model", models)
135
+@pytest.mark.parametrize(
136
+ "size_factors",
137
+ [
138
+ # No image
139
+ [],
140
+ # Single-scale
141
+ [1.0],
142
+ # Single-scale, batched
143
+ [1.0, 1.0, 1.0],
144
+ # Multi-scale
145
+ [0.25, 0.5, 1.0],
146
+ ],
147
+)
148
+@pytest.mark.parametrize("dtype", [target_dtype])
149
+@pytest.mark.parametrize("max_tokens", [128])
150
+@pytest.mark.parametrize("num_logprobs", [5])
151
+def test_models(hf_runner, vllm_runner, image_assets, model, size_factors,
152
+ dtype: str, max_tokens: int, num_logprobs: int) -> None:
153
+ run_test(
154
+ hf_runner,
155
+ vllm_runner,
156
+ image_assets,
157
+ model,
158
+ size_factors=size_factors,
159
+ dtype=dtype,
160
+ max_tokens=max_tokens,
161
+ num_logprobs=num_logprobs,
162
+ tensor_parallel_size=1,
163
+ )
vllm/model_executor/models/__init__.py
+1
@@ -50,6 +50,7 @@ _GENERATION_MODELS = {
50
"MptForCausalLM": ("mpt", "MPTForCausalLM"),
51
"MPTForCausalLM": ("mpt", "MPTForCausalLM"),
52
"MiniCPMForCausalLM": ("minicpm", "MiniCPMForCausalLM"),
53
+ "MiniCPMV": ("minicpmv", "MiniCPMV"),
54
"OlmoForCausalLM": ("olmo", "OlmoForCausalLM"),
55
"OPTForCausalLM": ("opt", "OPTForCausalLM"),
56
"OrionForCausalLM": ("orion", "OrionForCausalLM"),
vllm/model_executor/models/llama.py
+3
-1
@@ -418,9 +418,11 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA):
418
kv_caches: List[torch.Tensor],
419
attn_metadata: AttentionMetadata,
420
intermediate_tensors: Optional[IntermediateTensors] = None,
421
+ input_embeds: Optional[torch.Tensor] = None
422
) -> Union[torch.Tensor, IntermediateTensors]:
423
model_output = self.model(input_ids, positions, kv_caches,
423
- attn_metadata, intermediate_tensors)
424
+ attn_metadata, intermediate_tensors,
425
+ input_embeds)
426
return model_output
427
428
def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/minicpm.py
+2
-1
@@ -463,10 +463,11 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA):
463
positions: torch.Tensor,
464
kv_caches: List[torch.Tensor],
465
attn_metadata: AttentionMetadata,
466
+ input_embeds: Optional[torch.Tensor] = None,
467
intermediate_tensors: Optional[IntermediateTensors] = None,
468
) -> torch.Tensor:
469
hidden_states = self.model(input_ids, positions, kv_caches,
469
- attn_metadata)
470
+ attn_metadata, input_embeds)
471
return hidden_states
472
473
def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/minicpmv.py
new
+682
@@ -0,0 +1,682 @@
1
+# coding=utf-8
2
+# Adapted from
3
+# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
4
+# Copyright 2023 The vLLM team.
5
+# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
6
+#
7
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
8
+# and OPT implementations in this library. It has been modified from its
9
+# original forms to accommodate minor architectural differences compared
10
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
11
+#
12
+# Licensed under the Apache License, Version 2.0 (the "License");
13
+# you may not use this file except in compliance with the License.
14
+# You may obtain a copy of the License at
15
+#
16
+# http://www.apache.org/licenses/LICENSE-2.0
17
+#
18
+# Unless required by applicable law or agreed to in writing, software
19
+# distributed under the License is distributed on an "AS IS" BASIS,
20
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21
+# See the License for the specific language governing permissions and
22
+# limitations under the License.
23
+"""Inference-only MiniCPM-V-2 model compatible with HuggingFace weights."""
24
+import math
25
+import re
26
+from functools import partial
27
+from typing import Iterable, List, Optional, Tuple
28
+
29
+import numpy as np
30
+import torch
31
+import torch.nn.functional as F
32
+from PIL import Image
33
+from torch import nn
34
+from torch.nn.init import trunc_normal_
35
+from transformers.configuration_utils import PretrainedConfig
36
+from transformers.models.idefics2.modeling_idefics2 import (
37
+ Idefics2VisionTransformer)
38
+
39
+from vllm.attention import AttentionMetadata
40
+from vllm.config import CacheConfig, MultiModalConfig
41
+from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
42
+from vllm.model_executor.layers.quantization.base_config import (
43
+ QuantizationConfig)
44
+from vllm.model_executor.layers.sampler import Sampler
45
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
46
+from vllm.model_executor.models.interfaces import SupportsVision
47
+from vllm.model_executor.models.llama import LlamaForCausalLM
48
+from vllm.model_executor.models.minicpm import MiniCPMForCausalLM
49
+from vllm.model_executor.sampling_metadata import SamplingMetadata
50
+from vllm.multimodal import MULTIMODAL_REGISTRY
51
+from vllm.multimodal.image import (cached_get_image_processor,
52
+ cached_get_tokenizer)
53
+from vllm.sequence import IntermediateTensors, SamplerOutput, SequenceData
54
+
55
+_KEYS_TO_MODIFY_MAPPING = {
56
+ "language_model.lm_head": "lm_head",
57
+ "language_model.model": "language_model",
58
+}
59
+
60
+
61
+def get_abs_pos(abs_pos, tgt_size):
62
+ # abs_pos: L, C
63
+ # tgt_size: (H, W)
64
+ # return: M, C
65
+ src_size = int(math.sqrt(abs_pos.size(0)))
66
+ # tgt_size = int(math.sqrt(tgt_size))
67
+ dtype = abs_pos.dtype
68
+
69
+ return F.interpolate(
70
+ abs_pos.float().reshape(1, src_size, src_size, -1).permute(0, 3, 1, 2),
71
+ size=(tgt_size[0], tgt_size[1]),
72
+ mode="bicubic",
73
+ align_corners=False,
74
+ ).permute(0, 2, 3, 1).flatten(0, 2).to(dtype=dtype)
75
+
76
+
77
+# https://github.com/facebookresearch/mae/blob/efb2a8062c206524e35e47d04501ed4f544c0ae8/util/pos_embed.py#L20
78
+def get_2d_sincos_pos_embed(embed_dim,
79
+ grid_size,
80
+ cls_token=False,
81
+ version=2.0):
82
+ """
83
+ grid_size: int of the grid height and width
84
+ return:
85
+ pos_embed: [grid_size*grid_size, embed_dim] or
86
+ [1+grid_size*grid_size, embed_dim] (w/ or w/o cls_token)
87
+ """
88
+ if isinstance(grid_size, int):
89
+ grid_h_size, grid_w_size = grid_size, grid_size
90
+ else:
91
+ grid_h_size, grid_w_size = grid_size[0], grid_size[1]
92
+
93
+ grid_h = np.arange(grid_h_size, dtype=np.float32)
94
+ grid_w = np.arange(grid_w_size, dtype=np.float32)
95
+ grid = np.meshgrid(grid_w, grid_h) # here w goes first
96
+ grid = np.stack(grid, axis=0)
97
+
98
+ if version == 2.0:
99
+ grid = grid.reshape([2, 1, grid_h_size, grid_w_size])
100
+ pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid, version)
101
+ if cls_token:
102
+ pos_embed = np.concatenate([np.zeros([1, embed_dim]), pos_embed],
103
+ axis=0)
104
+ else:
105
+ pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid, version)
106
+ return pos_embed
107
+
108
+
109
+def get_2d_sincos_pos_embed_from_grid(embed_dim, grid, version=2.0):
110
+ assert embed_dim % 2 == 0
111
+
112
+ # use half of dimensions to encode grid_h
113
+ emb_h = get_1d_sincos_pos_embed_from_grid(
114
+ embed_dim // 2, grid[0], version) # (H*W, D/2) or (H, W, D/2)
115
+ emb_w = get_1d_sincos_pos_embed_from_grid(
116
+ embed_dim // 2, grid[1], version) # (H*W, D/2) or (H, W, D/2)
117
+
118
+ if version == 2.0:
119
+ emb = np.concatenate([emb_h, emb_w], axis=1) # (H*W, D)
120
+ else:
121
+ emb = np.concatenate([emb_h, emb_w], axis=-1) # (H, W, D)
122
+ return emb
123
+
124
+
125
+def get_1d_sincos_pos_embed_from_grid(embed_dim, pos, version=2.0):
126
+ """
127
+ embed_dim: output dimension for each position
128
+ pos: a list of positions to be encoded: size (M,) / (H, W)
129
+ out: (M, D) / (H, W, D)
130
+ """
131
+ assert embed_dim % 2 == 0
132
+ omega = np.arange(embed_dim // 2, dtype=np.float32)
133
+ omega /= embed_dim / 2.
134
+ omega = 1. / 10000**omega # (D/2,)
135
+
136
+ if version == 2.0:
137
+ pos = pos.reshape(-1) # (M,)
138
+ out = np.einsum('m,d->md', pos, omega) # (M, D/2), outer product
139
+ emb_sin = np.sin(out) # (M, D/2)
140
+ emb_cos = np.cos(out) # (M, D/2)
141
+ emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D)
142
+ else:
143
+ out = np.einsum('hw,d->hwd', pos, omega) # (H, W, D/2), outer product
144
+ emb_sin = np.sin(out) # (H, W, D/2)
145
+ emb_cos = np.cos(out) # (H, W, D/2)
146
+ emb = np.concatenate([emb_sin, emb_cos], axis=-1) # (H, W, D)
147
+ return emb
148
+
149
+
150
+class Resampler(nn.Module):
151
+ """
152
+ A 2D perceiver-resampler network with one cross attention layers by
153
+ (grid_size**2) learnable queries and 2d sincos pos_emb
154
+ Outputs:
155
+ A tensor with the shape of (grid_size**2, embed_dim)
156
+ """
157
+
158
+ default_norm_layer = partial(nn.LayerNorm, eps=1e-6)
159
+
160
+ def __init__(self,
161
+ num_queries,
162
+ grid_size,
163
+ embed_dim,
164
+ num_heads,
165
+ kv_dim=None,
166
+ norm_layer=default_norm_layer,
167
+ adaptive=False,
168
+ max_size=(70, 70),
169
+ version=2.0):
170
+ super().__init__()
171
+
172
+ self.version = version
173
+ if self.version == 2.0:
174
+ self.num_queries = grid_size**2
175
+ else:
176
+ self.num_queries = num_queries
177
+ self.max_size = max_size
178
+ self.embed_dim = embed_dim
179
+ self.num_heads = num_heads
180
+ self.adaptive = adaptive
181
+
182
+ self.query = nn.Parameter(torch.zeros(self.num_queries, embed_dim))
183
+ trunc_normal_(self.query, std=.02)
184
+
185
+ if kv_dim is not None and kv_dim != embed_dim:
186
+ self.kv_proj = nn.Linear(kv_dim, embed_dim, bias=False)
187
+ else:
188
+ self.kv_proj = nn.Identity()
189
+
190
+ self.attn = nn.MultiheadAttention(embed_dim, num_heads)
191
+ self.ln_q = norm_layer(embed_dim)
192
+ self.ln_kv = norm_layer(embed_dim)
193
+
194
+ self.ln_post = norm_layer(embed_dim)
195
+ self.proj = nn.Parameter(
196
+ (embed_dim**-0.5) * torch.randn(embed_dim, embed_dim))
197
+
198
+ if self.version == 2.0:
199
+ self.pos_embed = nn.Parameter(
200
+ torch.from_numpy(
201
+ get_2d_sincos_pos_embed(
202
+ embed_dim, grid_size,
203
+ version=self.version)).float()).requires_grad_(False)
204
+ else:
205
+ self._set_2d_pos_cache(self.max_size)
206
+
207
+ self.apply(self._init_weights)
208
+
209
+ def _set_2d_pos_cache(self, max_size, device='cpu'):
210
+ pos_embed = torch.from_numpy(
211
+ get_2d_sincos_pos_embed(self.embed_dim,
212
+ max_size,
213
+ version=self.version)).float().to(device)
214
+ self.register_buffer("pos_embed", pos_embed, persistent=False)
215
+
216
+ def _adjust_pos_cache(self, tgt_sizes, device):
217
+ max_h = torch.max(tgt_sizes[:, 0])
218
+ max_w = torch.max(tgt_sizes[:, 1])
219
+ if max_h > self.max_size[0] or max_w > self.max_size[1]:
220
+ self.max_size = [
221
+ max(max_h, self.max_size[0]),
222
+ max(max_w, self.max_size[1])
223
+ ]
224
+ self._set_2d_pos_cache(self.max_size, device)
225
+
226
+ def _init_weights(self, m):
227
+ if isinstance(m, nn.Linear):
228
+ trunc_normal_(m.weight, std=.02)
229
+ if isinstance(m, nn.Linear) and m.bias is not None:
230
+ nn.init.constant_(m.bias, 0)
231
+ elif isinstance(m, nn.LayerNorm):
232
+ nn.init.constant_(m.bias, 0)
233
+ nn.init.constant_(m.weight, 1.0)
234
+
235
+ def forward_2_5(self, x, tgt_sizes=None):
236
+ assert x.shape[0] == tgt_sizes.shape[0]
237
+ bs = x.shape[0]
238
+
239
+ device = x.device
240
+ dtype = x.dtype
241
+
242
+ patch_len = tgt_sizes[:, 0] * tgt_sizes[:, 1]
243
+
244
+ self._adjust_pos_cache(tgt_sizes, device=device)
245
+
246
+ max_patch_len = torch.max(patch_len)
247
+ key_padding_mask = torch.zeros((bs, max_patch_len),
248
+ dtype=torch.bool,
249
+ device=device)
250
+
251
+ pos_embed = []
252
+ for i in range(bs):
253
+ tgt_h, tgt_w = tgt_sizes[i]
254
+ pos_embed.append(self.pos_embed[:tgt_h, :tgt_w, :].reshape(
255
+ (tgt_h * tgt_w, -1)).to(dtype)) # patches * D
256
+ key_padding_mask[i, patch_len[i]:] = True
257
+
258
+ pos_embed = torch.nn.utils.rnn.pad_sequence(pos_embed,
259
+ batch_first=True,
260
+ padding_value=0.0).permute(
261
+ 1, 0,
262
+ 2) # BLD => L * B * D
263
+
264
+ x = self.kv_proj(x) # B * L * D
265
+ x = self.ln_kv(x).permute(1, 0, 2) # L * B * D
266
+
267
+ q = self.ln_q(self.query) # Q * D
268
+
269
+ out = self.attn(
270
+ self._repeat(q, bs), # Q * B * D
271
+ x + pos_embed, # L * B * D + L * B * D
272
+ x,
273
+ key_padding_mask=key_padding_mask)[0]
274
+ # out: Q * B * D
275
+ x = out.permute(1, 0, 2) # B * Q * D
276
+
277
+ x = self.ln_post(x)
278
+ x = x @ self.proj
279
+ return x
280
+
281
+ def forward_2(self, x, tgt_sizes=None, attn_mask=None):
282
+ if self.adaptive:
283
+ pos_embed = torch.Tensor(
284
+ get_2d_sincos_pos_embed(self.embed_dim,
285
+ tgt_sizes)).float().to(device=x.device,
286
+ dtype=x.dtype)
287
+ else:
288
+ pos_embed = get_abs_pos(self.pos_embed, tgt_sizes)
289
+
290
+ x = self.kv_proj(x)
291
+ x = self.ln_kv(x).permute(1, 0, 2)
292
+
293
+ N = x.shape[1]
294
+ q = self.ln_q(self.query)
295
+ out = self.attn(self._repeat(q, N) + self.pos_embed.unsqueeze(1),
296
+ x + pos_embed.unsqueeze(1),
297
+ x,
298
+ attn_mask=attn_mask)[0]
299
+ x = out.permute(1, 0, 2)
300
+
301
+ x = self.ln_post(x)
302
+ x = x @ self.proj
303
+ return x
304
+
305
+ def forward(self, x, tgt_sizes=None, attn_mask=None):
306
+ if self.version == 2.0:
307
+ return self.forward_2(x, tgt_sizes=tgt_sizes, attn_mask=attn_mask)
308
+ else:
309
+ return self.forward_2_5(x, tgt_sizes=tgt_sizes)
310
+
311
+ def _repeat(self, query, N: int):
312
+ return query.unsqueeze(1).repeat(1, N, 1)
313
+
314
+
315
+def get_max_minicpmv_image_tokens(ctx: InputContext):
316
+ hf_config = ctx.get_hf_config(PretrainedConfig)
317
+ return getattr(hf_config, "query_num", 64)
318
+
319
+
320
+def dummy_seq_data_for_minicpmv(seq_len: int):
321
+ token_ids = [0] * seq_len
322
+ return SequenceData(token_ids)
323
+
324
+
325
+def dummy_image_for_minicpmv(hf_config):
326
+ width = height = hf_config.image_size
327
+ image = Image.new("RGB", (width, height), color=0)
328
+ return {"image": image}
329
+
330
+
331
+def dummy_data_for_minicpmv(ctx: InputContext, seq_len: int):
332
+ hf_config = ctx.get_hf_config(PretrainedConfig)
333
+
334
+ # image_feature_size = get_max_minicpmv_image_tokens(ctx)
335
+
336
+ seq_data = dummy_seq_data_for_minicpmv(seq_len)
337
+
338
+ mm_data = dummy_image_for_minicpmv(hf_config)
339
+
340
+ return seq_data, mm_data
341
+
342
+
343
+def input_processor_for_minicpmv(ctx: InputContext, llm_inputs: LLMInputs):
344
+ multi_modal_data = llm_inputs.get("multi_modal_data")
345
+ if multi_modal_data is None or "image" not in multi_modal_data:
346
+ return llm_inputs
347
+
348
+ model_config = ctx.model_config
349
+
350
+ tokenizer = cached_get_tokenizer(model_config.tokenizer,
351
+ trust_remote_code=True)
352
+
353
+ prompt = llm_inputs.get("prompt")
354
+ if prompt is None:
355
+ token_ids = llm_inputs.get("prompt_token_ids")
356
+ prompt = tokenizer.decode(token_ids)
357
+ image_processor = cached_get_image_processor(model_config.tokenizer)
358
+
359
+ pattern = "(<image>./</image>)"
360
+ image = multi_modal_data["image"]
361
+ image_tags = re.findall(pattern, prompt)
362
+ assert len(image_tags) <= 1
363
+ text_chunks = prompt.split(pattern)
364
+ new_prompt = text_chunks[0] \
365
+ + image_processor.get_slice_image_placeholder(image.size) \
366
+ + text_chunks[1]
367
+
368
+ new_token_ids = tokenizer.encode(new_prompt)
369
+
370
+ llm_inputs = LLMInputs(prompt_token_ids=new_token_ids,
371
+ prompt=new_prompt,
372
+ multi_modal_data=multi_modal_data)
373
+ return llm_inputs
374
+
375
+
376
+@MULTIMODAL_REGISTRY.register_image_input_mapper()
377
+@MULTIMODAL_REGISTRY.register_max_image_tokens(get_max_minicpmv_image_tokens)
378
+@INPUT_REGISTRY.register_dummy_data(dummy_data_for_minicpmv)
379
+@INPUT_REGISTRY.register_input_processor(input_processor_for_minicpmv)
380
+class MiniCPMV(nn.Module, SupportsVision):
381
+
382
+ def __init__(
383
+ self,
384
+ config,
385
+ multimodal_config: MultiModalConfig,
386
+ cache_config: Optional[CacheConfig] = None,
387
+ quant_config: Optional[QuantizationConfig] = None,
388
+ ):
389
+ super().__init__()
390
+ self.config = config
391
+ self.multimodal_config = multimodal_config
392
+
393
+ self.version = float(self.config.version)
394
+ self.llm = self.init_llm(config, cache_config, quant_config)
395
+ self.vpm = self.init_vision_module()
396
+ param_dtype = torch.get_default_dtype()
397
+ self.vpm.to(dtype=param_dtype)
398
+ self.vision_dim = self.vpm.embed_dim if self.version == 2.0 \
399
+ else self.vpm.embeddings.embed_dim
400
+ self.embed_dim = self.llm.config.hidden_size
401
+ self.resampler = self.init_resampler(self.embed_dim, self.vision_dim)
402
+ self.resampler.to(device="cuda", dtype=param_dtype)
403
+ self.sampler = Sampler()
404
+
405
+ def init_llm(self, config, cache_config, quant_config):
406
+ if self.version == 2.0:
407
+ return MiniCPMForCausalLM(config,
408
+ cache_config=cache_config,
409
+ quant_config=quant_config)
410
+ else:
411
+ return LlamaForCausalLM(config,
412
+ cache_config=cache_config,
413
+ quant_config=quant_config)
414
+
415
+ def init_vision_module(self):
416
+ if self.version == 2.0:
417
+ try:
418
+ import timm
419
+ except ImportError:
420
+ raise ImportError(
421
+ 'Please install timm==0.9.10') from ImportError
422
+ default_dtype = torch.get_default_dtype()
423
+ torch.set_default_dtype(torch.float16)
424
+ model = timm.create_model('vit_so400m_patch14_siglip_384.webli',
425
+ pretrained=False,
426
+ num_classes=0,
427
+ dynamic_img_size=True,
428
+ dynamic_img_pad=True)
429
+ torch.set_default_dtype(default_dtype)
430
+ if isinstance(model, timm.models.VisionTransformer
431
+ ) and model.attn_pool is not None:
432
+ model.attn_pool = torch.nn.Identity()
433
+
434
+ if self.config.drop_vision_last_layer:
435
+ model.blocks = model.blocks[:-1]
436
+ else:
437
+ model = Idefics2VisionTransformer(self.config.vision_config)
438
+ if self.config.drop_vision_last_layer:
439
+ model.encoder.layers = model.encoder.layers[:-1]
440
+ return model
441
+
442
+ def init_resampler(self, embed_dim, vision_dim):
443
+ default_dtype = torch.get_default_dtype()
444
+ torch.set_default_dtype(torch.float16)
445
+ if self.version == 2.0:
446
+ resampler = Resampler(grid_size=int(
447
+ math.sqrt(self.config.query_num)),
448
+ num_queries=None,
449
+ embed_dim=embed_dim,
450
+ num_heads=embed_dim // 128,
451
+ kv_dim=vision_dim,
452
+ adaptive=True,
453
+ version=self.version)
454
+ else:
455
+ resampler = Resampler(num_queries=self.config.query_num,
456
+ grid_size=None,
457
+ embed_dim=embed_dim,
458
+ num_heads=embed_dim // 128,
459
+ kv_dim=vision_dim,
460
+ adaptive=True,
461
+ version=self.version)
462
+ torch.set_default_dtype(default_dtype)
463
+ return resampler
464
+
465
+ def get_vision_embedding(self,
466
+ pixel_values,
467
+ patch_attn_mask=None,
468
+ tgt_sizes=None,
469
+ version=2.0):
470
+ if version == 2.0:
471
+ res = []
472
+ dtype = self.vpm.pos_embed.data.dtype
473
+ for pixel_value in pixel_values:
474
+ # V2.0 start
475
+ H, W = pixel_value[0].shape[-2:]
476
+ tgt_size = (math.ceil(H / self.vpm.patch_embed.patch_size[0]),
477
+ math.ceil(W / self.vpm.patch_embed.patch_size[0]))
478
+ # V2.0 end
479
+ vision_embedding = self.vpm.forward_features(
480
+ pixel_value.unsqueeze(0).type(dtype))
481
+ if hasattr(self.vpm, 'num_prefix_tokens'
482
+ ) and self.vpm.num_prefix_tokens > 0:
483
+ vision_embedding = vision_embedding[:, self.vpm.
484
+ num_prefix_tokens:]
485
+ res.append(self.resampler(vision_embedding, tgt_size))
486
+ return torch.vstack(res)
487
+ else:
488
+ vision_embedding = self.vpm(
489
+ pixel_values.type(dtype),
490
+ patch_attention_mask=patch_attn_mask).last_hidden_state
491
+ vision_embedding = self.resampler(vision_embedding, tgt_sizes)
492
+
493
+ def get_image_bounds(self, input_ids):
494
+ tokenizer = cached_get_tokenizer(self.config._name_or_path,
495
+ trust_remote_code=True)
496
+ im_start_token_id = tokenizer.im_start_id
497
+ im_end_token_id = tokenizer.im_end_id
498
+ image_start_tokens = torch.where(input_ids == im_start_token_id)[0]
499
+ image_start_tokens += 1
500
+ image_end_tokens = torch.where(input_ids == im_end_token_id)[0]
501
+ valid_image_nums = min(len(image_start_tokens), len(image_end_tokens))
502
+ if valid_image_nums == 0:
503
+ return []
504
+ image_bound = torch.hstack([
505
+ image_start_tokens[:valid_image_nums].unsqueeze(-1),
506
+ image_end_tokens[:valid_image_nums].unsqueeze(-1),
507
+ ])
508
+
509
+ return image_bound
510
+
511
+ def get_vision_hidden_states(self, data):
512
+ if "vision_hidden_states" not in data:
513
+ pixel_values = data["pixel_values"]
514
+ tgt_sizes = data["tgt_sizes"]
515
+ vision_hidden_states = []
516
+ if self.version == 2.0:
517
+ if pixel_values is not None and len(pixel_values) > 0:
518
+ vision_hidden_states = self.get_vision_embedding(
519
+ pixel_values)
520
+ else:
521
+ vision_hidden_states = torch.tensor([]).to(
522
+ data["input_ids"].device)
523
+ else:
524
+ device = self.vpm.embeddings.position_embedding.weight.device
525
+ dtype = self.vpm.embeddings.position_embedding.weight.dtype
526
+ all_pixel_values = [
527
+ i.flatten(end_dim=1).permute(1, 0) for i in pixel_values
528
+ ]
529
+ if all_pixel_values:
530
+ tgt_sizes = torch.vstack(tgt_sizes).type(torch.int32)
531
+ max_patches = torch.max(tgt_sizes[:, 0] * tgt_sizes[:, 1])
532
+ all_pixel_values = torch.nn.utils.rnn.pad_sequence(
533
+ all_pixel_values, batch_first=True, padding_value=0.0)
534
+ B, L, _ = all_pixel_values.shape
535
+ all_pixel_values = all_pixel_values.permute(
536
+ 0, 2, 1).reshape(B, 3, -1, L)
537
+
538
+ patch_attn_mask = torch.zeros((B, 1, max_patches),
539
+ dtype=torch.bool,
540
+ device=device)
541
+ for i in range(B):
542
+ patch_attn_mask[i, :tgt_sizes[i][0] *
543
+ tgt_sizes[i][1]] = True
544
+
545
+ vision_embedding = self.vpm(
546
+ all_pixel_values.type(dtype),
547
+ patch_attention_mask=patch_attn_mask).last_hidden_state
548
+ vision_hidden_states = self.resampler(
549
+ vision_embedding, tgt_sizes)
550
+
551
+ else: # no image
552
+ dummy_feature = []
553
+ vision_hidden_states = dummy_feature
554
+ else:
555
+ vision_hidden_states = data["vision_hidden_states"]
556
+
557
+ return vision_hidden_states
558
+
559
+ def get_embedding(self, data):
560
+ input_ids = data["input_ids"]
561
+
562
+ vision_hidden_states = self.get_vision_hidden_states(data)
563
+ if vision_hidden_states is not None and len(vision_hidden_states) > 0:
564
+ image_bounds = self.get_image_bounds(input_ids)
565
+ else:
566
+ image_bounds = []
567
+
568
+ if hasattr(self.llm.config, 'scale_emb'):
569
+ vlm_embedding = self.llm.model.embed_tokens(
570
+ input_ids) * self.llm.config.scale_emb
571
+ else:
572
+ vlm_embedding = self.llm.model.embed_tokens(input_ids)
573
+ vision_hidden_states = [
574
+ i.type(vlm_embedding.dtype) if isinstance(i, torch.Tensor) else i
575
+ for i in vision_hidden_states
576
+ ]
577
+
578
+ if len(vision_hidden_states) > 0 and len(image_bounds) > 0:
579
+ vision_hidden_states = torch.cat(vision_hidden_states, dim=0)
580
+ image_indices = torch.stack([
581
+ torch.arange(r[0], r[1], dtype=torch.long)
582
+ for r in image_bounds
583
+ ]).to(vlm_embedding.device)
584
+ vlm_embedding.scatter_(
585
+ 0,
586
+ image_indices.view(-1, 1).repeat(1, vlm_embedding.shape[-1]),
587
+ vision_hidden_states.view(-1, vision_hidden_states.shape[-1]))
588
+ return vlm_embedding, vision_hidden_states
589
+
590
+ def process_multimodal_inputs(self, inputs):
591
+ pixel_values = []
592
+ tgt_sizes = []
593
+ for b in range(len(inputs["pixel_values"])):
594
+ pixel_values += inputs["pixel_values"][b]
595
+ tgt_sizes += inputs["tgt_sizes"][b]
596
+ return {
597
+ "pixel_values": pixel_values,
598
+ "input_ids": inputs["input_ids"],
599
+ "tgt_sizes": tgt_sizes
600
+ }
601
+
602
+ def forward(
603
+ self,
604
+ input_ids: torch.Tensor,
605
+ positions: torch.Tensor,
606
+ kv_caches: List[torch.Tensor],
607
+ attn_metadata: AttentionMetadata,
608
+ intermediate_tensors: Optional[IntermediateTensors] = None,
609
+ **kwargs: object,
610
+ ):
611
+ inputs = {
612
+ "pixel_values": kwargs.pop("pixel_values", []),
613
+ "input_ids": input_ids,
614
+ "tgt_sizes": kwargs.pop("tgt_sizes", None),
615
+ }
616
+
617
+ inputs = self.process_multimodal_inputs(inputs)
618
+
619
+ vlm_embeddings, vision_hidden_states = self.get_embedding(inputs)
620
+
621
+ output = self.llm(input_ids=None,
622
+ positions=positions,
623
+ kv_caches=kv_caches,
624
+ attn_metadata=attn_metadata,
625
+ intermediate_tensors=intermediate_tensors,
626
+ input_embeds=vlm_embeddings)
627
+ return output
628
+
629
+ def compute_logits(self, hidden_states: torch.Tensor,
630
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
631
+ return self.llm.compute_logits(hidden_states, sampling_metadata)
632
+
633
+ def sample(
634
+ self,
635
+ logits: torch.Tensor,
636
+ sampling_metadata: SamplingMetadata,
637
+ ) -> Optional[SamplerOutput]:
638
+ next_tokens = self.llm.sample(logits, sampling_metadata)
639
+ return next_tokens
640
+
641
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
642
+ stacked_params_mapping = [
643
+ # (param_name, shard_name, shard_id)
644
+ ("qkv_proj", "q_proj", "q"),
645
+ ("qkv_proj", "k_proj", "k"),
646
+ ("qkv_proj", "v_proj", "v"),
647
+ ("gate_up_proj", "gate_proj", 0),
648
+ ("gate_up_proj", "up_proj", 1),
649
+ ]
650
+ params_dict = dict(self.named_parameters())
651
+ for name, loaded_weight in weights:
652
+ # for key_to_modify, new_key in _KEYS_TO_MODIFY_MAPPING.items():
653
+ # if key_to_modify in name:
654
+ # name = name.replace(key_to_modify, new_key)
655
+ if "rotary_emb.inv_freq" in name:
656
+ continue
657
+ if ("rotary_emb.cos_cached" in name
658
+ or "rotary_emb.sin_cached" in name):
659
+ # Models trained using ColossalAI may include these tensors in
660
+ # the checkpoint. Skip them.
661
+ continue
662
+ use_default_weight_loading = False
663
+ if "vpm" in name or 'resampler' in name:
664
+ # We only do sharding for language model and
665
+ # not vision model for now.
666
+ use_default_weight_loading = True
667
+ else:
668
+ for (param_name, weight_name,
669
+ shard_id) in stacked_params_mapping:
670
+ if weight_name not in name:
671
+ continue
672
+ param = params_dict[name.replace(weight_name, param_name)]
673
+ weight_loader = param.weight_loader
674
+ weight_loader(param, loaded_weight, shard_id)
675
+ break
676
+ else:
677
+ use_default_weight_loading = True
678
+ if use_default_weight_loading:
679
+ param = params_dict[name]
680
+ weight_loader = getattr(param, "weight_loader",
681
+ default_weight_loader)
682
+ weight_loader(param, loaded_weight)
vllm/multimodal/__init__.py
+2
-1
@@ -1,5 +1,5 @@
1
from .base import (BatchedTensors, MultiModalDataBuiltins, MultiModalDataDict,
2
- MultiModalInputs, MultiModalPlugin)
2
+ MultiModalInputs, MultiModalPlugin, NestedTensors)
3
from .registry import MultiModalRegistry
4
5
MULTIMODAL_REGISTRY = MultiModalRegistry()
@@ -17,6 +17,7 @@ __all__ = [
17
"MultiModalDataDict",
18
"MultiModalInputs",
19
"MultiModalPlugin",
20
+ "NestedTensors",
21
"MULTIMODAL_REGISTRY",
22
"MultiModalRegistry",
23
]
vllm/multimodal/base.py
+24
-10
@@ -2,7 +2,7 @@ import sys
2
from abc import ABC, abstractmethod
3
from collections import UserDict, defaultdict
4
from typing import (Any, Callable, Dict, List, Optional, Type, TypedDict,
5
- TypeVar, Union)
5
+ TypeVar, Union, cast)
6
7
import torch
8
import torch.types
@@ -15,10 +15,17 @@ from vllm.logger import init_logger
15
16
logger = init_logger(__name__)
17
18
-BatchedTensors = Union[torch.Tensor, List[torch.Tensor]]
18
+NestedTensors = Union[List[torch.Tensor], torch.Tensor]
19
+"""
20
+Use a list instead of a tensor if the dimensions of each element do not match.
21
+Currently only supports up to singly nested list of tensors.
22
+"""
23
+
24
+BatchedTensors = Union[List[NestedTensors], NestedTensors]
25
"""
26
If each input tensor in the batch has the same size, this is a single batched
21
-tensor; otherwise, this is a list of tensors with one element per batch.
27
+tensor; otherwise, this is a list of :class:`NestedTensors` with one element
28
+per item in the batch.
29
"""
30
31
if sys.version_info < (3, 9):
@@ -27,7 +34,7 @@ if sys.version_info < (3, 9):
34
pass
35
else:
36
30
- class _MultiModalInputsBase(UserDict[str, torch.Tensor]):
37
+ class _MultiModalInputsBase(UserDict[str, NestedTensors]):
38
pass
39
40
@@ -39,19 +46,26 @@ class MultiModalInputs(_MultiModalInputsBase):
46
47
@staticmethod
48
def try_concat(
42
- tensors: List[torch.Tensor],
49
+ tensors: List[NestedTensors],
50
*,
51
device: torch.types.Device,
52
) -> BatchedTensors:
46
- unbatched_shape = tensors[0].shape[1:]
53
+ # may be list rather than tensors
54
+ if isinstance(tensors[0], list):
55
+ return [[t.to(device=device) for t in tensor[0]]
56
+ for tensor in tensors]
57
+
58
+ tensors_ = cast(List[torch.Tensor], tensors)
59
+
60
+ unbatched_shape = tensors_[0].shape[1:]
61
48
- for tensor in tensors:
62
+ for tensor in tensors_:
63
if tensor.shape[1:] != unbatched_shape:
64
return [
51
- tensor.squeeze(0).to(device=device) for tensor in tensors
65
+ tensor.squeeze(0).to(device=device) for tensor in tensors_
66
]
67
54
- return torch.cat(tensors, dim=0).to(device=device)
68
+ return torch.cat(tensors_, dim=0).to(device=device)
69
70
@staticmethod
71
def batch(
@@ -64,7 +78,7 @@ class MultiModalInputs(_MultiModalInputsBase):
78
79
keys = inputs_list[0].keys()
80
67
- item_lists: Dict[str, List[torch.Tensor]] = defaultdict(list)
81
+ item_lists: Dict[str, List[NestedTensors]] = defaultdict(list)
82
83
for inputs in inputs_list:
84
if inputs.keys() != keys: