A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Bugfix] Clean up MiniCPM-V (#6939)
Co-authored-by: hezhihui <hzh7269@modelbest.cn> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
Alphi committed
Jul 31, 2024 at 22:39 UTC
2f4e108f75c817bf2f323e306db590e13d2863f6
6 files changed
+975
-94
docs/source/models/supported_models.rst
+5
-1
@@ -222,9 +222,13 @@ Vision Language Models
222
-
223
* - :code:`MiniCPM-V`
224
- MiniCPM-V
225
- - :code:`openbmb/MiniCPM-V-2`, :code:`openbmb/MiniCPM-Llama3-V-2_5`, etc.
225
+ - :code:`openbmb/MiniCPM-V-2` (see note), :code:`openbmb/MiniCPM-Llama3-V-2_5`, etc.
226
-
227
228
+.. note::
229
+ For :code:`openbmb/MiniCPM-V-2`, the official repo doesn't work yet, so we need to use a fork (:code:`HwwwH/MiniCPM-V-2`) for now.
230
+ For more details, please see: https://github.com/vllm-project/vllm/pull/4087#issuecomment-2250397630
231
+
232
----
233
234
If your model uses one of the above model architectures, you can seamlessly run your model with vLLM.
vllm/model_executor/models/llama.py
+1
-3
@@ -418,11 +418,9 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA):
418
kv_caches: List[torch.Tensor],
419
attn_metadata: AttentionMetadata,
420
intermediate_tensors: Optional[IntermediateTensors] = None,
421
- input_embeds: Optional[torch.Tensor] = None
421
) -> Union[torch.Tensor, IntermediateTensors]:
422
model_output = self.model(input_ids, positions, kv_caches,
424
- attn_metadata, intermediate_tensors,
425
- input_embeds)
423
+ attn_metadata, intermediate_tensors)
424
return model_output
425
426
def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/minicpm.py
+2
-2
@@ -370,6 +370,7 @@ class MiniCPMModel(nn.Module):
370
positions: torch.Tensor,
371
kv_caches: List[torch.Tensor],
372
attn_metadata: AttentionMetadata,
373
+ intermediate_tensors: Optional[IntermediateTensors] = None,
374
inputs_embeds: Optional[torch.Tensor] = None,
375
) -> torch.Tensor:
376
if inputs_embeds is not None:
@@ -463,11 +464,10 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA):
464
positions: torch.Tensor,
465
kv_caches: List[torch.Tensor],
466
attn_metadata: AttentionMetadata,
466
- input_embeds: Optional[torch.Tensor] = None,
467
intermediate_tensors: Optional[IntermediateTensors] = None,
468
) -> torch.Tensor:
469
hidden_states = self.model(input_ids, positions, kv_caches,
470
- attn_metadata, input_embeds)
470
+ attn_metadata, intermediate_tensors)
471
return hidden_states
472
473
def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/minicpmv.py
+162
-87
@@ -20,32 +20,34 @@
20
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21
# See the License for the specific language governing permissions and
22
# limitations under the License.
23
-"""Inference-only MiniCPM-V-2 model compatible with HuggingFace weights."""
23
+"""Inference-only MiniCPM-V model compatible with HuggingFace weights."""
24
import math
25
import re
26
from functools import partial
27
-from typing import Iterable, List, Optional, Tuple
27
+from typing import Dict, Iterable, List, Optional, Tuple, Union
28
29
import numpy as np
30
import torch
31
import torch.nn.functional as F
32
+import torch.types
33
from PIL import Image
34
from torch import nn
35
from torch.nn.init import trunc_normal_
36
from transformers.configuration_utils import PretrainedConfig
36
-from transformers.models.idefics2.modeling_idefics2 import (
37
- Idefics2VisionTransformer)
37
38
from vllm.attention import AttentionMetadata
39
from vllm.config import CacheConfig, MultiModalConfig
40
from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
41
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
42
from vllm.model_executor.layers.quantization.base_config import (
43
QuantizationConfig)
44
from vllm.model_executor.layers.sampler import Sampler
45
+from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
46
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
47
from vllm.model_executor.models.interfaces import SupportsVision
47
-from vllm.model_executor.models.llama import LlamaForCausalLM
48
-from vllm.model_executor.models.minicpm import MiniCPMForCausalLM
48
+from vllm.model_executor.models.llama import LlamaModel
49
+from vllm.model_executor.models.minicpm import MiniCPMModel
50
+from vllm.model_executor.models.qwen2 import Qwen2Model
51
from vllm.model_executor.sampling_metadata import SamplingMetadata
52
from vllm.multimodal import MULTIMODAL_REGISTRY
53
from vllm.multimodal.image import (cached_get_image_processor,
@@ -53,12 +55,12 @@ from vllm.multimodal.image import (cached_get_image_processor,
55
from vllm.sequence import IntermediateTensors, SamplerOutput, SequenceData
56
57
_KEYS_TO_MODIFY_MAPPING = {
56
- "language_model.lm_head": "lm_head",
57
- "language_model.model": "language_model",
58
+ "llm.lm_head": "lm_head",
59
+ "llm.model": "llm",
60
}
61
62
61
-def get_abs_pos(abs_pos, tgt_size):
63
+def get_abs_pos(abs_pos: torch.Tensor, tgt_size: torch.Tensor):
64
# abs_pos: L, C
65
# tgt_size: (H, W)
66
# return: M, C
@@ -75,10 +77,10 @@ def get_abs_pos(abs_pos, tgt_size):
77
78
79
# https://github.com/facebookresearch/mae/blob/efb2a8062c206524e35e47d04501ed4f544c0ae8/util/pos_embed.py#L20
78
-def get_2d_sincos_pos_embed(embed_dim,
79
- grid_size,
80
- cls_token=False,
81
- version=2.0):
80
+def get_2d_sincos_pos_embed(embed_dim: int,
81
+ grid_size: Union[int, Tuple[int, int]],
82
+ cls_token: bool = False,
83
+ version: Tuple[int, int] = (2, 0)):
84
"""
85
grid_size: int of the grid height and width
86
return:
@@ -95,7 +97,7 @@ def get_2d_sincos_pos_embed(embed_dim,
97
grid = np.meshgrid(grid_w, grid_h) # here w goes first
98
grid = np.stack(grid, axis=0)
99
98
- if version == 2.0:
100
+ if version == (2, 0):
101
grid = grid.reshape([2, 1, grid_h_size, grid_w_size])
102
pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid, version)
103
if cls_token:
@@ -106,7 +108,9 @@ def get_2d_sincos_pos_embed(embed_dim,
108
return pos_embed
109
110
109
-def get_2d_sincos_pos_embed_from_grid(embed_dim, grid, version=2.0):
111
+def get_2d_sincos_pos_embed_from_grid(embed_dim: int,
112
+ grid: Union[int, Tuple[int, int]],
113
+ version: Tuple[int, int] = (2, 0)):
114
assert embed_dim % 2 == 0
115
116
# use half of dimensions to encode grid_h
@@ -115,14 +119,16 @@ def get_2d_sincos_pos_embed_from_grid(embed_dim, grid, version=2.0):
119
emb_w = get_1d_sincos_pos_embed_from_grid(
120
embed_dim // 2, grid[1], version) # (H*W, D/2) or (H, W, D/2)
121
118
- if version == 2.0:
122
+ if version == (2, 0):
123
emb = np.concatenate([emb_h, emb_w], axis=1) # (H*W, D)
124
else:
125
emb = np.concatenate([emb_h, emb_w], axis=-1) # (H, W, D)
126
return emb
127
128
125
-def get_1d_sincos_pos_embed_from_grid(embed_dim, pos, version=2.0):
129
+def get_1d_sincos_pos_embed_from_grid(embed_dim: int,
130
+ pos: int,
131
+ version: Tuple[int, int] = (2, 0)):
132
"""
133
embed_dim: output dimension for each position
134
pos: a list of positions to be encoded: size (M,) / (H, W)
@@ -133,7 +139,7 @@ def get_1d_sincos_pos_embed_from_grid(embed_dim, pos, version=2.0):
139
omega /= embed_dim / 2.
140
omega = 1. / 10000**omega # (D/2,)
141
136
- if version == 2.0:
142
+ if version == (2, 0):
143
pos = pos.reshape(-1) # (M,)
144
out = np.einsum('m,d->md', pos, omega) # (M, D/2), outer product
145
emb_sin = np.sin(out) # (M, D/2)
@@ -158,19 +164,19 @@ class Resampler(nn.Module):
164
default_norm_layer = partial(nn.LayerNorm, eps=1e-6)
165
166
def __init__(self,
161
- num_queries,
162
- grid_size,
163
- embed_dim,
164
- num_heads,
165
- kv_dim=None,
166
- norm_layer=default_norm_layer,
167
- adaptive=False,
168
- max_size=(70, 70),
169
- version=2.0):
167
+ num_queries: int,
168
+ grid_size: int,
169
+ embed_dim: int,
170
+ num_heads: int,
171
+ kv_dim: Optional[int] = None,
172
+ norm_layer: nn.Module = default_norm_layer,
173
+ adaptive: bool = False,
174
+ max_size: Tuple[int, int] = (70, 70),
175
+ version: Tuple[int, int] = (2, 0)):
176
super().__init__()
177
178
self.version = version
173
- if self.version == 2.0:
179
+ if self.version == (2, 0):
180
self.num_queries = grid_size**2
181
else:
182
self.num_queries = num_queries
@@ -195,7 +201,7 @@ class Resampler(nn.Module):
201
self.proj = nn.Parameter(
202
(embed_dim**-0.5) * torch.randn(embed_dim, embed_dim))
203
198
- if self.version == 2.0:
204
+ if self.version == (2, 0):
205
self.pos_embed = nn.Parameter(
206
torch.from_numpy(
207
get_2d_sincos_pos_embed(
@@ -206,14 +212,17 @@ class Resampler(nn.Module):
212
213
self.apply(self._init_weights)
214
209
- def _set_2d_pos_cache(self, max_size, device='cpu'):
215
+ def _set_2d_pos_cache(self,
216
+ max_size: Tuple[int, int],
217
+ device: torch.types.Device = 'cpu'):
218
pos_embed = torch.from_numpy(
219
get_2d_sincos_pos_embed(self.embed_dim,
220
max_size,
221
version=self.version)).float().to(device)
222
self.register_buffer("pos_embed", pos_embed, persistent=False)
223
216
- def _adjust_pos_cache(self, tgt_sizes, device):
224
+ def _adjust_pos_cache(self, tgt_sizes: torch.Tensor,
225
+ device: torch.types.Device):
226
max_h = torch.max(tgt_sizes[:, 0])
227
max_w = torch.max(tgt_sizes[:, 1])
228
if max_h > self.max_size[0] or max_w > self.max_size[1]:
@@ -223,7 +232,7 @@ class Resampler(nn.Module):
232
]
233
self._set_2d_pos_cache(self.max_size, device)
234
226
- def _init_weights(self, m):
235
+ def _init_weights(self, m: nn.Module):
236
if isinstance(m, nn.Linear):
237
trunc_normal_(m.weight, std=.02)
238
if isinstance(m, nn.Linear) and m.bias is not None:
@@ -232,7 +241,9 @@ class Resampler(nn.Module):
241
nn.init.constant_(m.bias, 0)
242
nn.init.constant_(m.weight, 1.0)
243
235
- def forward_2_5(self, x, tgt_sizes=None):
244
+ def forward_2_5(self,
245
+ x: torch.Tensor,
246
+ tgt_sizes: Optional[torch.Tensor] = None):
247
assert x.shape[0] == tgt_sizes.shape[0]
248
bs = x.shape[0]
249
@@ -278,7 +289,10 @@ class Resampler(nn.Module):
289
x = x @ self.proj
290
return x
291
281
- def forward_2(self, x, tgt_sizes=None, attn_mask=None):
292
+ def forward_2(self,
293
+ x: torch.Tensor,
294
+ tgt_sizes: Optional[torch.Tensor] = None,
295
+ attn_mask: Optional[torch.Tensor] = None):
296
if self.adaptive:
297
pos_embed = torch.Tensor(
298
get_2d_sincos_pos_embed(self.embed_dim,
@@ -302,8 +316,11 @@ class Resampler(nn.Module):
316
x = x @ self.proj
317
return x
318
305
- def forward(self, x, tgt_sizes=None, attn_mask=None):
306
- if self.version == 2.0:
319
+ def forward(self,
320
+ x: torch.Tensor,
321
+ tgt_sizes: Optional[torch.Tensor] = None,
322
+ attn_mask: Optional[torch.Tensor] = None):
323
+ if self.version == (2, 0):
324
return self.forward_2(x, tgt_sizes=tgt_sizes, attn_mask=attn_mask)
325
else:
326
return self.forward_2_5(x, tgt_sizes=tgt_sizes)
@@ -322,7 +339,7 @@ def dummy_seq_data_for_minicpmv(seq_len: int):
339
return SequenceData(token_ids)
340
341
325
-def dummy_image_for_minicpmv(hf_config):
342
+def dummy_image_for_minicpmv(hf_config: PretrainedConfig):
343
width = height = hf_config.image_size
344
image = Image.new("RGB", (width, height), color=0)
345
return {"image": image}
@@ -381,7 +398,7 @@ class MiniCPMV(nn.Module, SupportsVision):
398
399
def __init__(
400
self,
384
- config,
401
+ config: PretrainedConfig,
402
multimodal_config: MultiModalConfig,
403
cache_config: Optional[CacheConfig] = None,
404
quant_config: Optional[QuantizationConfig] = None,
@@ -390,30 +407,48 @@ class MiniCPMV(nn.Module, SupportsVision):
407
self.config = config
408
self.multimodal_config = multimodal_config
409
393
- self.version = float(self.config.version)
410
+ if not hasattr(self.config, "version"):
411
+ if self.config.hidden_size == 2304 and self.config.query_num == 64:
412
+ self.version = (2, 0)
413
+ else:
414
+ self.version = (2, 5)
415
+ else:
416
+ self.version = str(self.config.version).split(".")
417
+ self.version = tuple([int(x) for x in self.version])
418
self.llm = self.init_llm(config, cache_config, quant_config)
419
self.vpm = self.init_vision_module()
420
param_dtype = torch.get_default_dtype()
421
self.vpm.to(dtype=param_dtype)
398
- self.vision_dim = self.vpm.embed_dim if self.version == 2.0 \
422
+ self.vision_dim = self.vpm.embed_dim if self.version == (2, 0) \
423
else self.vpm.embeddings.embed_dim
400
- self.embed_dim = self.llm.config.hidden_size
424
+ self.embed_dim = self.config.hidden_size
425
self.resampler = self.init_resampler(self.embed_dim, self.vision_dim)
426
self.resampler.to(device="cuda", dtype=param_dtype)
427
+ self.lm_head = ParallelLMHead(config.vocab_size,
428
+ config.hidden_size,
429
+ quant_config=quant_config)
430
+ self.logits_processor = LogitsProcessor(config.vocab_size)
431
self.sampler = Sampler()
432
405
- def init_llm(self, config, cache_config, quant_config):
406
- if self.version == 2.0:
407
- return MiniCPMForCausalLM(config,
408
- cache_config=cache_config,
409
- quant_config=quant_config)
433
+ def init_llm(self,
434
+ config: PretrainedConfig,
435
+ cache_config: Optional[CacheConfig] = None,
436
+ quant_config: Optional[QuantizationConfig] = None):
437
+ if self.version == (2, 0):
438
+ return MiniCPMModel(config,
439
+ cache_config=cache_config,
440
+ quant_config=quant_config)
441
+ elif self.version == (2, 5):
442
+ return LlamaModel(config,
443
+ cache_config=cache_config,
444
+ quant_config=quant_config)
445
else:
411
- return LlamaForCausalLM(config,
412
- cache_config=cache_config,
413
- quant_config=quant_config)
446
+ return Qwen2Model(config,
447
+ cache_config=cache_config,
448
+ quant_config=quant_config)
449
450
def init_vision_module(self):
416
- if self.version == 2.0:
451
+ if self.version == (2, 0):
452
try:
453
import timm
454
except ImportError:
@@ -433,16 +468,30 @@ class MiniCPMV(nn.Module, SupportsVision):
468
469
if self.config.drop_vision_last_layer:
470
model.blocks = model.blocks[:-1]
436
- else:
471
+ elif self.version == (2, 5):
472
+ from transformers.models.idefics2.modeling_idefics2 import (
473
+ Idefics2VisionTransformer)
474
model = Idefics2VisionTransformer(self.config.vision_config)
475
if self.config.drop_vision_last_layer:
476
model.encoder.layers = model.encoder.layers[:-1]
477
+ else:
478
+ from vllm.model_executor.models.na_vit import (
479
+ SiglipVisionTransformer)
480
+ if self.config._attn_implementation == 'flash_attention_2':
481
+ self.config.vision_config._attn_implementation \
482
+ = 'flash_attention_2'
483
+ else:
484
+ # not support sdpa
485
+ self.config.vision_config._attn_implementation = 'eager'
486
+ model = SiglipVisionTransformer(self.config.vision_config)
487
+ if self.config.drop_vision_last_layer:
488
+ model.encoder.layers = model.encoder.layers[:-1]
489
return model
490
442
- def init_resampler(self, embed_dim, vision_dim):
491
+ def init_resampler(self, embed_dim: int, vision_dim: int):
492
default_dtype = torch.get_default_dtype()
493
torch.set_default_dtype(torch.float16)
445
- if self.version == 2.0:
494
+ if self.version == (2, 0):
495
resampler = Resampler(grid_size=int(
496
math.sqrt(self.config.query_num)),
497
num_queries=None,
@@ -463,11 +512,11 @@ class MiniCPMV(nn.Module, SupportsVision):
512
return resampler
513
514
def get_vision_embedding(self,
466
- pixel_values,
467
- patch_attn_mask=None,
468
- tgt_sizes=None,
469
- version=2.0):
470
- if version == 2.0:
515
+ pixel_values: List[List[torch.Tensor]],
516
+ patch_attn_mask: Optional[torch.Tensor] = None,
517
+ tgt_sizes: Optional[torch.Tensor] = None,
518
+ version: Tuple[int, int] = (2, 0)):
519
+ if version == (2, 0):
520
res = []
521
dtype = self.vpm.pos_embed.data.dtype
522
for pixel_value in pixel_values:
@@ -484,21 +533,32 @@ class MiniCPMV(nn.Module, SupportsVision):
533
num_prefix_tokens:]
534
res.append(self.resampler(vision_embedding, tgt_size))
535
return torch.vstack(res)
487
- else:
536
+ elif version == (2, 5):
537
vision_embedding = self.vpm(
538
pixel_values.type(dtype),
539
patch_attention_mask=patch_attn_mask).last_hidden_state
540
vision_embedding = self.resampler(vision_embedding, tgt_sizes)
541
+ else:
542
+ vision_embedding = self.vpm(pixel_values.type(dtype),
543
+ patch_attention_mask=patch_attn_mask,
544
+ tgt_sizes=tgt_sizes).last_hidden_state
545
493
- def get_image_bounds(self, input_ids):
546
+ def get_image_bounds(self, input_ids: torch.Tensor):
547
tokenizer = cached_get_tokenizer(self.config._name_or_path,
548
trust_remote_code=True)
496
- im_start_token_id = tokenizer.im_start_id
497
- im_end_token_id = tokenizer.im_end_id
498
- image_start_tokens = torch.where(input_ids == im_start_token_id)[0]
549
+ if not hasattr(tokenizer, "slice_start_id"):
550
+ start_cond = input_ids == tokenizer.im_start_id
551
+ end_cond = input_ids == tokenizer.im_end_id
552
+ else:
553
+ start_cond = (input_ids == tokenizer.im_start_id) | (
554
+ input_ids == tokenizer.slice_start_id)
555
+ end_cond = (input_ids == tokenizer.im_end_id) | (
556
+ input_ids == tokenizer.slice_end_id)
557
+
558
+ image_start_tokens = torch.where(start_cond)[0]
559
image_start_tokens += 1
500
- image_end_tokens = torch.where(input_ids == im_end_token_id)[0]
501
- valid_image_nums = min(len(image_start_tokens), len(image_end_tokens))
560
+ image_end_tokens = torch.where(end_cond)[0]
561
+ valid_image_nums = max(len(image_start_tokens), len(image_end_tokens))
562
if valid_image_nums == 0:
563
return []
564
image_bound = torch.hstack([
@@ -508,12 +568,14 @@ class MiniCPMV(nn.Module, SupportsVision):
568
569
return image_bound
570
511
- def get_vision_hidden_states(self, data):
571
+ def get_vision_hidden_states(self, data: Dict[str,
572
+ Union[List[torch.Tensor],
573
+ torch.Tensor]]):
574
if "vision_hidden_states" not in data:
575
pixel_values = data["pixel_values"]
576
tgt_sizes = data["tgt_sizes"]
577
vision_hidden_states = []
516
- if self.version == 2.0:
578
+ if self.version == (2, 0):
579
if pixel_values is not None and len(pixel_values) > 0:
580
vision_hidden_states = self.get_vision_embedding(
581
pixel_values)
@@ -534,17 +596,26 @@ class MiniCPMV(nn.Module, SupportsVision):
596
B, L, _ = all_pixel_values.shape
597
all_pixel_values = all_pixel_values.permute(
598
0, 2, 1).reshape(B, 3, -1, L)
537
-
599
patch_attn_mask = torch.zeros((B, 1, max_patches),
600
dtype=torch.bool,
601
device=device)
541
- for i in range(B):
542
- patch_attn_mask[i, :tgt_sizes[i][0] *
543
- tgt_sizes[i][1]] = True
602
+ if self.version == (2, 5):
603
+ for i in range(B):
604
+ patch_attn_mask[i, :tgt_sizes[i][0] *
605
+ tgt_sizes[i][1]] = True
606
+ vision_embedding = self.vpm(
607
+ all_pixel_values.type(dtype),
608
+ patch_attention_mask=patch_attn_mask
609
+ ).last_hidden_state
610
+ else:
611
+ for i in range(B):
612
+ patch_attn_mask[i, 0, :tgt_sizes[i][0] *
613
+ tgt_sizes[i][1]] = True
614
+ vision_embedding = self.vpm(
615
+ all_pixel_values.type(dtype),
616
+ patch_attention_mask=patch_attn_mask,
617
+ tgt_sizes=tgt_sizes).last_hidden_state
618
545
- vision_embedding = self.vpm(
546
- all_pixel_values.type(dtype),
547
- patch_attention_mask=patch_attn_mask).last_hidden_state
619
vision_hidden_states = self.resampler(
620
vision_embedding, tgt_sizes)
621
@@ -556,7 +627,8 @@ class MiniCPMV(nn.Module, SupportsVision):
627
628
return vision_hidden_states
629
559
- def get_embedding(self, data):
630
+ def get_embedding(self, data: Dict[str, Union[List[torch.Tensor],
631
+ torch.Tensor]]):
632
input_ids = data["input_ids"]
633
634
vision_hidden_states = self.get_vision_hidden_states(data)
@@ -565,11 +637,11 @@ class MiniCPMV(nn.Module, SupportsVision):
637
else:
638
image_bounds = []
639
568
- if hasattr(self.llm.config, 'scale_emb'):
569
- vlm_embedding = self.llm.model.embed_tokens(
570
- input_ids) * self.llm.config.scale_emb
640
+ if hasattr(self.config, 'scale_emb'):
641
+ vlm_embedding = self.llm.embed_tokens(
642
+ input_ids) * self.config.scale_emb
643
else:
572
- vlm_embedding = self.llm.model.embed_tokens(input_ids)
644
+ vlm_embedding = self.llm.embed_tokens(input_ids)
645
vision_hidden_states = [
646
i.type(vlm_embedding.dtype) if isinstance(i, torch.Tensor) else i
647
for i in vision_hidden_states
@@ -587,7 +659,9 @@ class MiniCPMV(nn.Module, SupportsVision):
659
vision_hidden_states.view(-1, vision_hidden_states.shape[-1]))
660
return vlm_embedding, vision_hidden_states
661
590
- def process_multimodal_inputs(self, inputs):
662
+ def process_multimodal_inputs(self, inputs: Dict[str,
663
+ Union[List[torch.Tensor],
664
+ torch.Tensor]]):
665
pixel_values = []
666
tgt_sizes = []
667
for b in range(len(inputs["pixel_values"])):
@@ -613,7 +687,6 @@ class MiniCPMV(nn.Module, SupportsVision):
687
"input_ids": input_ids,
688
"tgt_sizes": kwargs.pop("tgt_sizes", None),
689
}
616
-
690
inputs = self.process_multimodal_inputs(inputs)
691
692
vlm_embeddings, vision_hidden_states = self.get_embedding(inputs)
@@ -623,19 +696,21 @@ class MiniCPMV(nn.Module, SupportsVision):
696
kv_caches=kv_caches,
697
attn_metadata=attn_metadata,
698
intermediate_tensors=intermediate_tensors,
626
- input_embeds=vlm_embeddings)
699
+ inputs_embeds=vlm_embeddings)
700
return output
701
702
def compute_logits(self, hidden_states: torch.Tensor,
703
sampling_metadata: SamplingMetadata) -> torch.Tensor:
631
- return self.llm.compute_logits(hidden_states, sampling_metadata)
704
+ logits = self.logits_processor(self.lm_head, hidden_states,
705
+ sampling_metadata)
706
+ return logits
707
708
def sample(
709
self,
710
logits: torch.Tensor,
711
sampling_metadata: SamplingMetadata,
712
) -> Optional[SamplerOutput]:
638
- next_tokens = self.llm.sample(logits, sampling_metadata)
713
+ next_tokens = self.sampler(logits, sampling_metadata)
714
return next_tokens
715
716
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
@@ -649,9 +724,9 @@ class MiniCPMV(nn.Module, SupportsVision):
724
]
725
params_dict = dict(self.named_parameters())
726
for name, loaded_weight in weights:
652
- # for key_to_modify, new_key in _KEYS_TO_MODIFY_MAPPING.items():
653
- # if key_to_modify in name:
654
- # name = name.replace(key_to_modify, new_key)
727
+ for key_to_modify, new_key in _KEYS_TO_MODIFY_MAPPING.items():
728
+ if key_to_modify in name:
729
+ name = name.replace(key_to_modify, new_key)
730
if "rotary_emb.inv_freq" in name:
731
continue
732
if ("rotary_emb.cos_cached" in name
vllm/model_executor/models/na_vit.py
new
+804
@@ -0,0 +1,804 @@
1
+import logging
2
+import math
3
+import os
4
+import warnings
5
+from typing import Optional, Tuple, Union
6
+
7
+import numpy as np
8
+import torch
9
+import torch.nn.functional as F
10
+from torch import nn
11
+from torch.nn.init import _calculate_fan_in_and_fan_out
12
+from transformers.activations import ACT2FN
13
+from transformers.configuration_utils import PretrainedConfig
14
+from transformers.modeling_attn_mask_utils import _prepare_4d_attention_mask
15
+from transformers.modeling_outputs import (BaseModelOutput,
16
+ BaseModelOutputWithPooling)
17
+from transformers.modeling_utils import PreTrainedModel
18
+from transformers.utils import (ModelOutput, is_flash_attn_2_available,
19
+ replace_return_docstrings)
20
+
21
+logger = logging.getLogger("vllm")
22
+
23
+
24
+# For Siglip: copied from
25
+# HuggingFaceM4/siglip-so400m-14-980-flash-attn2-navit and add tgt_sizes
26
+# Remove hints as there's little possibility to change these code.
27
+class SiglipVisionConfig(PretrainedConfig):
28
+
29
+ model_type = "siglip_vision_model"
30
+
31
+ def __init__(
32
+ self,
33
+ hidden_size=768,
34
+ intermediate_size=3072,
35
+ num_hidden_layers=12,
36
+ num_attention_heads=12,
37
+ num_channels=3,
38
+ image_size=224,
39
+ patch_size=16,
40
+ hidden_act="gelu_pytorch_tanh",
41
+ layer_norm_eps=1e-6,
42
+ attention_dropout=0.0,
43
+ **kwargs,
44
+ ):
45
+ super().__init__(**kwargs)
46
+
47
+ self.hidden_size = hidden_size
48
+ self.intermediate_size = intermediate_size
49
+ self.num_hidden_layers = num_hidden_layers
50
+ self.num_attention_heads = num_attention_heads
51
+ self.num_channels = num_channels
52
+ self.patch_size = patch_size
53
+ self.image_size = image_size
54
+ self.attention_dropout = attention_dropout
55
+ self.layer_norm_eps = layer_norm_eps
56
+ self.hidden_act = hidden_act
57
+
58
+ @classmethod
59
+ def from_pretrained(cls, pretrained_model_name_or_path: Union[str,
60
+ os.PathLike],
61
+ **kwargs) -> "PretrainedConfig":
62
+ cls._set_token_in_kwargs(kwargs)
63
+
64
+ config_dict, kwargs = cls.get_config_dict(
65
+ pretrained_model_name_or_path, **kwargs)
66
+
67
+ # get the vision config dict if we are loading from SiglipConfig
68
+ if config_dict.get("model_type") == "siglip":
69
+ config_dict = config_dict["vision_config"]
70
+
71
+ if "model_type" in config_dict and hasattr(
72
+ cls,
73
+ "model_type") and config_dict["model_type"] != cls.model_type:
74
+ logger.warning(
75
+ "You are using a model of type %s to "
76
+ "instantiate a model of type %s. "
77
+ "This is not supported for all configurations"
78
+ "of models and can yield errors.", config_dict['model_type'],
79
+ cls.model_type)
80
+
81
+ return cls.from_dict(config_dict, **kwargs)
82
+
83
+
84
+_CHECKPOINT_FOR_DOC = "google/siglip-base-patch16-224"
85
+
86
+SIGLIP_PRETRAINED_MODEL_ARCHIVE_LIST = [
87
+ "google/siglip-base-patch16-224",
88
+ # See all SigLIP models at https://huggingface.co/models?filter=siglip
89
+]
90
+
91
+if is_flash_attn_2_available():
92
+ from flash_attn import flash_attn_func, flash_attn_varlen_func
93
+ from flash_attn.bert_padding import pad_input # noqa
94
+ from flash_attn.bert_padding import index_first_axis, unpad_input
95
+
96
+
97
+# Copied from transformers.models.llama.modeling_llama._get_unpad_data
98
+def _get_unpad_data(attention_mask):
99
+ seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32)
100
+ indices = torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten()
101
+ max_seqlen_in_batch = seqlens_in_batch.max().item()
102
+ cu_seqlens = F.pad(
103
+ torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.torch.int32), (1, 0))
104
+ return (
105
+ indices,
106
+ cu_seqlens,
107
+ max_seqlen_in_batch,
108
+ )
109
+
110
+
111
+def _trunc_normal_(tensor, mean, std, a, b):
112
+
113
+ def norm_cdf(x):
114
+ # Computes standard normal cumulative distribution function
115
+ return (1.0 + math.erf(x / math.sqrt(2.0))) / 2.0
116
+
117
+ if (mean < a - 2 * std) or (mean > b + 2 * std):
118
+ warnings.warn(
119
+ "mean is more than 2 std from [a, b] in nn.init.trunc_normal_. "
120
+ "The distribution of values may be incorrect.",
121
+ stacklevel=2,
122
+ )
123
+
124
+ # Values are generated by using a truncated uniform distribution and
125
+ # then using the inverse CDF for the normal distribution.
126
+ # Get upper and lower cdf values
127
+ l_ = norm_cdf((a - mean) / std)
128
+ u = norm_cdf((b - mean) / std)
129
+
130
+ # Uniformly fill tensor with values from [l, u], then translate to
131
+ # [2l-1, 2u-1].
132
+ tensor.uniform_(2 * l_ - 1, 2 * u - 1)
133
+
134
+ # Use inverse cdf transform for normal distribution to get truncated
135
+ # standard normal
136
+ if tensor.dtype in [torch.float16, torch.bfloat16]:
137
+ # The `erfinv_` op is not (yet?) defined in float16+cpu, bfloat16+gpu
138
+ og_dtype = tensor.dtype
139
+ tensor = tensor.to(torch.float32)
140
+ tensor.erfinv_()
141
+ tensor = tensor.to(og_dtype)
142
+ else:
143
+ tensor.erfinv_()
144
+
145
+ # Transform to proper mean, std
146
+ tensor.mul_(std * math.sqrt(2.0))
147
+ tensor.add_(mean)
148
+
149
+ # Clamp to ensure it's in the proper range
150
+ if tensor.dtype == torch.float16:
151
+ # The `clamp_` op is not (yet?) defined in float16+cpu
152
+ tensor = tensor.to(torch.float32)
153
+ tensor.clamp_(min=a, max=b)
154
+ tensor = tensor.to(torch.float16)
155
+ else:
156
+ tensor.clamp_(min=a, max=b)
157
+
158
+
159
+def trunc_normal_tf_(tensor: torch.Tensor,
160
+ mean: float = 0.0,
161
+ std: float = 1.0,
162
+ a: float = -2.0,
163
+ b: float = 2.0) -> torch.Tensor:
164
+ with torch.no_grad():
165
+ _trunc_normal_(tensor, 0, 1.0, a, b)
166
+ tensor.mul_(std).add_(mean)
167
+
168
+
169
+def variance_scaling_(tensor, scale=1.0, mode="fan_in", distribution="normal"):
170
+ fan_in, fan_out = _calculate_fan_in_and_fan_out(tensor)
171
+ if mode == "fan_in":
172
+ denom = fan_in
173
+ elif mode == "fan_out":
174
+ denom = fan_out
175
+ elif mode == "fan_avg":
176
+ denom = (fan_in + fan_out) / 2
177
+
178
+ variance = scale / denom
179
+
180
+ if distribution == "truncated_normal":
181
+ # constant is stddev of standard normal truncated to (-2, 2)
182
+ trunc_normal_tf_(tensor, std=math.sqrt(variance) / 0.87962566103423978)
183
+ elif distribution == "normal":
184
+ with torch.no_grad():
185
+ tensor.normal_(std=math.sqrt(variance))
186
+ elif distribution == "uniform":
187
+ bound = math.sqrt(3 * variance)
188
+ with torch.no_grad():
189
+ tensor.uniform_(-bound, bound)
190
+ else:
191
+ raise ValueError(f"invalid distribution {distribution}")
192
+
193
+
194
+def lecun_normal_(tensor):
195
+ variance_scaling_(tensor, mode="fan_in", distribution="truncated_normal")
196
+
197
+
198
+def default_flax_embed_init(tensor):
199
+ variance_scaling_(tensor, mode="fan_in", distribution="normal")
200
+
201
+
202
+class SiglipVisionModelOutput(ModelOutput):
203
+ image_embeds: Optional[torch.FloatTensor] = None
204
+ last_hidden_state: torch.FloatTensor = None
205
+ hidden_states: Optional[Tuple[torch.FloatTensor]] = None
206
+ attentions: Optional[Tuple[torch.FloatTensor]] = None
207
+
208
+
209
+class SiglipVisionEmbeddings(nn.Module):
210
+
211
+ def __init__(self, config: SiglipVisionConfig):
212
+ super().__init__()
213
+ self.config = config
214
+ self.embed_dim = config.hidden_size
215
+ self.image_size = config.image_size
216
+ self.patch_size = config.patch_size
217
+
218
+ self.patch_embedding = nn.Conv2d(
219
+ in_channels=config.num_channels,
220
+ out_channels=self.embed_dim,
221
+ kernel_size=self.patch_size,
222
+ stride=self.patch_size,
223
+ padding="valid",
224
+ )
225
+
226
+ self.num_patches_per_side = self.image_size // self.patch_size
227
+ self.num_patches = self.num_patches_per_side**2
228
+ self.num_positions = self.num_patches
229
+ self.position_embedding = nn.Embedding(self.num_positions,
230
+ self.embed_dim)
231
+
232
+ def forward(self,
233
+ pixel_values: torch.FloatTensor,
234
+ patch_attention_mask: torch.BoolTensor,
235
+ tgt_sizes: Optional[torch.IntTensor] = None) -> torch.Tensor:
236
+ batch_size = pixel_values.size(0)
237
+
238
+ patch_embeds = self.patch_embedding(pixel_values)
239
+ embeddings = patch_embeds.flatten(2).transpose(1, 2)
240
+
241
+ max_im_h, max_im_w = pixel_values.size(2), pixel_values.size(3)
242
+ max_nb_patches_h, max_nb_patches_w = (max_im_h // self.patch_size,
243
+ max_im_w // self.patch_size)
244
+ boundaries = torch.arange(1 / self.num_patches_per_side, 1.0,
245
+ 1 / self.num_patches_per_side)
246
+ position_ids = torch.full(
247
+ size=(
248
+ batch_size,
249
+ max_nb_patches_h * max_nb_patches_w,
250
+ ),
251
+ fill_value=0,
252
+ )
253
+
254
+ for batch_idx, p_attn_mask in enumerate(patch_attention_mask):
255
+ if tgt_sizes is not None:
256
+ nb_patches_h = tgt_sizes[batch_idx][0]
257
+ nb_patches_w = tgt_sizes[batch_idx][1]
258
+ else:
259
+ nb_patches_h = p_attn_mask[:, 0].sum()
260
+ nb_patches_w = p_attn_mask[0].sum()
261
+
262
+ fractional_coords_h = torch.arange(0, 1 - 1e-6, 1 / nb_patches_h)
263
+ fractional_coords_w = torch.arange(0, 1 - 1e-6, 1 / nb_patches_w)
264
+
265
+ bucket_coords_h = torch.bucketize(fractional_coords_h,
266
+ boundaries,
267
+ right=True)
268
+ bucket_coords_w = torch.bucketize(fractional_coords_w,
269
+ boundaries,
270
+ right=True)
271
+
272
+ pos_ids = (bucket_coords_h[:, None] * self.num_patches_per_side +
273
+ bucket_coords_w).flatten()
274
+ position_ids[batch_idx][p_attn_mask.view(-1).cpu()] = pos_ids
275
+
276
+ position_ids = position_ids.to(self.position_embedding.weight.device)
277
+
278
+ embeddings = embeddings + self.position_embedding(position_ids)
279
+ return embeddings
280
+
281
+
282
+class SiglipAttention(nn.Module):
283
+ """Multi-headed attention from 'Attention Is All You Need' paper"""
284
+
285
+ # Copied from transformers.models.clip.modeling_clip.CLIPAttention.__init__
286
+ def __init__(self, config):
287
+ super().__init__()
288
+ self.config = config
289
+ self.embed_dim = config.hidden_size
290
+ self.num_heads = config.num_attention_heads
291
+ self.head_dim = self.embed_dim // self.num_heads
292
+ if self.head_dim * self.num_heads != self.embed_dim:
293
+ raise ValueError(
294
+ "embed_dim must be divisible by num_heads (got `embed_dim`: "
295
+ f"{self.embed_dim} and `num_heads`:"
296
+ f" {self.num_heads}).")
297
+ self.scale = self.head_dim**-0.5
298
+ self.dropout = config.attention_dropout
299
+
300
+ self.k_proj = nn.Linear(self.embed_dim, self.embed_dim)
301
+ self.v_proj = nn.Linear(self.embed_dim, self.embed_dim)
302
+ self.q_proj = nn.Linear(self.embed_dim, self.embed_dim)
303
+ self.out_proj = nn.Linear(self.embed_dim, self.embed_dim)
304
+
305
+ def forward(
306
+ self,
307
+ hidden_states: torch.Tensor,
308
+ attention_mask: Optional[torch.Tensor] = None,
309
+ output_attentions: Optional[bool] = False,
310
+ ) -> Tuple[torch.Tensor, Optional[torch.Tensor],
311
+ Optional[Tuple[torch.Tensor]]]:
312
+ """Input shape: Batch x Time x Channel"""
313
+
314
+ batch_size, q_len, _ = hidden_states.size()
315
+
316
+ query_states = self.q_proj(hidden_states)
317
+ key_states = self.k_proj(hidden_states)
318
+ value_states = self.v_proj(hidden_states)
319
+
320
+ query_states = query_states.view(batch_size, q_len, self.num_heads,
321
+ self.head_dim).transpose(1, 2)
322
+ key_states = key_states.view(batch_size, q_len, self.num_heads,
323
+ self.head_dim).transpose(1, 2)
324
+ value_states = value_states.view(batch_size, q_len, self.num_heads,
325
+ self.head_dim).transpose(1, 2)
326
+
327
+ k_v_seq_len = key_states.shape[-2]
328
+ attn_weights = torch.matmul(query_states, key_states.transpose(
329
+ 2, 3)) * self.scale
330
+
331
+ if attn_weights.size() != (batch_size, self.num_heads, q_len,
332
+ k_v_seq_len):
333
+ raise ValueError(
334
+ "Attention weights should be of size "
335
+ f"{(batch_size, self.num_heads, q_len, k_v_seq_len)}, but is"
336
+ f" {attn_weights.size()}")
337
+
338
+ if attention_mask is not None:
339
+ if attention_mask.size() != (batch_size, 1, q_len, k_v_seq_len):
340
+ raise ValueError(
341
+ "Attention mask should be of size "
342
+ f"{(batch_size, 1, q_len, k_v_seq_len)}",
343
+ f"but is {attention_mask.size()}")
344
+ attn_weights = attn_weights + attention_mask
345
+
346
+ # upcast attention to fp32
347
+ attn_weights = nn.functional.softmax(attn_weights,
348
+ dim=-1,
349
+ dtype=torch.float32).to(
350
+ query_states.dtype)
351
+ attn_weights = nn.functional.dropout(attn_weights,
352
+ p=self.dropout,
353
+ training=self.training)
354
+ attn_output = torch.matmul(attn_weights, value_states)
355
+
356
+ if attn_output.size() != (batch_size, self.num_heads, q_len,
357
+ self.head_dim):
358
+ raise ValueError(
359
+ "`attn_output` should be of size "
360
+ f"{(batch_size, self.num_heads, q_len, self.head_dim)}, "
361
+ "but is"
362
+ f" {attn_output.size()}")
363
+
364
+ attn_output = attn_output.transpose(1, 2).contiguous()
365
+ attn_output = attn_output.reshape(batch_size, q_len, self.embed_dim)
366
+
367
+ attn_output = self.out_proj(attn_output)
368
+
369
+ return attn_output, attn_weights
370
+
371
+
372
+class SiglipFlashAttention2(SiglipAttention):
373
+
374
+ def __init__(self, *args, **kwargs):
375
+ super().__init__(*args, **kwargs)
376
+ self.is_causal = False # Hack to make sure we don't use a causal mask
377
+
378
+ def forward(
379
+ self,
380
+ hidden_states: torch.Tensor,
381
+ attention_mask: Optional[torch.LongTensor] = None,
382
+ position_ids: Optional[torch.LongTensor] = None,
383
+ past_key_value: Optional[Tuple[torch.Tensor]] = None,
384
+ output_attentions: bool = False,
385
+ use_cache: bool = False,
386
+ **kwargs,
387
+ ) -> Tuple[torch.Tensor, Optional[torch.Tensor],
388
+ Optional[Tuple[torch.Tensor]]]:
389
+ output_attentions = False
390
+
391
+ bsz, q_len, _ = hidden_states.size()
392
+
393
+ query_states = self.q_proj(hidden_states)
394
+ key_states = self.k_proj(hidden_states)
395
+ value_states = self.v_proj(hidden_states)
396
+
397
+ query_states = query_states.view(bsz, q_len, self.num_heads,
398
+ self.head_dim).transpose(1, 2)
399
+ key_states = key_states.view(bsz, q_len, self.num_heads,
400
+ self.head_dim).transpose(1, 2)
401
+ value_states = value_states.view(bsz, q_len, self.num_heads,
402
+ self.head_dim).transpose(1, 2)
403
+
404
+ kv_seq_len = key_states.shape[-2]
405
+ if past_key_value is not None:
406
+ kv_seq_len += past_key_value.get_usable_length(
407
+ kv_seq_len, self.layer_idx)
408
+
409
+ query_states = query_states.transpose(1, 2)
410
+ key_states = key_states.transpose(1, 2)
411
+ value_states = value_states.transpose(1, 2)
412
+
413
+ dropout_rate = self.dropout if self.training else 0.0
414
+
415
+ input_dtype = query_states.dtype
416
+ if input_dtype == torch.float32:
417
+ if torch.is_autocast_enabled():
418
+ target_dtype = torch.get_autocast_gpu_dtype()
419
+ # Handle the case where the model is quantized
420
+ elif hasattr(self.config, "_pre_quantization_dtype"):
421
+ target_dtype = self.config._pre_quantization_dtype
422
+ else:
423
+ target_dtype = self.q_proj.weight.dtype
424
+
425
+ logger.warning(
426
+ "The input hidden states seems to be "
427
+ "silently casted in float32, "
428
+ "this might be related to the fact "
429
+ "you have upcasted embedding or layer norm layers in float32. "
430
+ "We will cast back the input in"
431
+ " %s.", target_dtype)
432
+
433
+ query_states = query_states.to(target_dtype)
434
+ key_states = key_states.to(target_dtype)
435
+ value_states = value_states.to(target_dtype)
436
+
437
+ attn_output = self._flash_attention_forward(query_states,
438
+ key_states,
439
+ value_states,
440
+ attention_mask,
441
+ q_len,
442
+ dropout=dropout_rate)
443
+
444
+ attn_output = attn_output.reshape(bsz, q_len,
445
+ self.embed_dim).contiguous()
446
+ attn_output = self.out_proj(attn_output)
447
+
448
+ if not output_attentions:
449
+ attn_weights = None
450
+
451
+ return attn_output, attn_weights
452
+
453
+ def _flash_attention_forward(self,
454
+ query_states,
455
+ key_states,
456
+ value_states,
457
+ attention_mask,
458
+ query_length,
459
+ dropout=0.0,
460
+ softmax_scale=None):
461
+ causal = self.is_causal and query_length != 1
462
+
463
+ # Contains at least one padding token in the sequence
464
+ if attention_mask is not None:
465
+ batch_size = query_states.shape[0]
466
+ (query_states, key_states, value_states, indices_q, cu_seq_lens,
467
+ max_seq_lens) = self._upad_input(query_states, key_states,
468
+ value_states, attention_mask,
469
+ query_length)
470
+
471
+ cu_seqlens_q, cu_seqlens_k = cu_seq_lens
472
+ max_seqlen_in_batch_q, max_seqlen_in_batch_k = max_seq_lens
473
+
474
+ attn_output_unpad = flash_attn_varlen_func(
475
+ query_states,
476
+ key_states,
477
+ value_states,
478
+ cu_seqlens_q=cu_seqlens_q,
479
+ cu_seqlens_k=cu_seqlens_k,
480
+ max_seqlen_q=max_seqlen_in_batch_q,
481
+ max_seqlen_k=max_seqlen_in_batch_k,
482
+ dropout_p=dropout,
483
+ softmax_scale=softmax_scale,
484
+ causal=causal,
485
+ )
486
+
487
+ attn_output = pad_input(attn_output_unpad, indices_q, batch_size,
488
+ query_length)
489
+ else:
490
+ attn_output = flash_attn_func(query_states,
491
+ key_states,
492
+ value_states,
493
+ dropout,
494
+ softmax_scale=softmax_scale,
495
+ causal=causal)
496
+
497
+ return attn_output
498
+
499
+ def _upad_input(self, query_layer, key_layer, value_layer, attention_mask,
500
+ query_length):
501
+ indices_k, cu_seqlens_k, max_seqlen_in_batch_k = _get_unpad_data(
502
+ attention_mask)
503
+ batch_size, kv_seq_len, num_key_value_heads, head_dim = key_layer.shape
504
+
505
+ key_layer = index_first_axis(
506
+ key_layer.reshape(batch_size * kv_seq_len, num_key_value_heads,
507
+ head_dim), indices_k)
508
+ value_layer = index_first_axis(
509
+ value_layer.reshape(batch_size * kv_seq_len, num_key_value_heads,
510
+ head_dim), indices_k)
511
+ if query_length == kv_seq_len:
512
+ query_layer = index_first_axis(
513
+ query_layer.reshape(batch_size * kv_seq_len, self.num_heads,
514
+ head_dim), indices_k)
515
+ cu_seqlens_q = cu_seqlens_k
516
+ max_seqlen_in_batch_q = max_seqlen_in_batch_k
517
+ indices_q = indices_k
518
+ elif query_length == 1:
519
+ max_seqlen_in_batch_q = 1
520
+ cu_seqlens_q = torch.arange(
521
+ batch_size + 1, dtype=torch.int32, device=query_layer.device
522
+ ) # There is a memcpy here, that is very bad.
523
+ indices_q = cu_seqlens_q[:-1]
524
+ query_layer = query_layer.squeeze(1)
525
+ else:
526
+ # The -q_len: slice assumes left padding.
527
+ attention_mask = attention_mask[:, -query_length:]
528
+ (query_layer, indices_q, cu_seqlens_q,
529
+ max_seqlen_in_batch_q) = unpad_input(query_layer, attention_mask)
530
+
531
+ return (
532
+ query_layer,
533
+ key_layer,
534
+ value_layer,
535
+ indices_q,
536
+ (cu_seqlens_q, cu_seqlens_k),
537
+ (max_seqlen_in_batch_q, max_seqlen_in_batch_k),
538
+ )
539
+
540
+
541
+# Copied from transformers.models.clip.modeling_clip.CLIPMLP with CLIP->Siglip
542
+class SiglipMLP(nn.Module):
543
+
544
+ def __init__(self, config):
545
+ super().__init__()
546
+ self.config = config
547
+ self.activation_fn = ACT2FN[config.hidden_act]
548
+ self.fc1 = nn.Linear(config.hidden_size, config.intermediate_size)
549
+ self.fc2 = nn.Linear(config.intermediate_size, config.hidden_size)
550
+
551
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
552
+ hidden_states = self.fc1(hidden_states)
553
+ hidden_states = self.activation_fn(hidden_states)
554
+ hidden_states = self.fc2(hidden_states)
555
+ return hidden_states
556
+
557
+
558
+# Copied from transformers.models.clip.modeling_clip.CLIPEncoderLayer
559
+# with CLIP->Siglip
560
+class SiglipEncoderLayer(nn.Module):
561
+
562
+ def __init__(self, config: SiglipVisionConfig):
563
+ super().__init__()
564
+ self.embed_dim = config.hidden_size
565
+ self._use_flash_attention_2 = (
566
+ config._attn_implementation == "flash_attention_2")
567
+ self.self_attn = (SiglipAttention(config)
568
+ if not self._use_flash_attention_2 else
569
+ SiglipFlashAttention2(config))
570
+ self.layer_norm1 = nn.LayerNorm(self.embed_dim,
571
+ eps=config.layer_norm_eps)
572
+ self.mlp = SiglipMLP(config)
573
+ self.layer_norm2 = nn.LayerNorm(self.embed_dim,
574
+ eps=config.layer_norm_eps)
575
+
576
+ def forward(
577
+ self,
578
+ hidden_states: torch.Tensor,
579
+ attention_mask: torch.Tensor,
580
+ output_attentions: Optional[bool] = False,
581
+ ) -> Tuple[torch.FloatTensor]:
582
+ residual = hidden_states
583
+
584
+ hidden_states = self.layer_norm1(hidden_states)
585
+ hidden_states, attn_weights = self.self_attn(
586
+ hidden_states=hidden_states,
587
+ attention_mask=attention_mask,
588
+ output_attentions=output_attentions,
589
+ )
590
+ hidden_states = residual + hidden_states
591
+
592
+ residual = hidden_states
593
+ hidden_states = self.layer_norm2(hidden_states)
594
+ hidden_states = self.mlp(hidden_states)
595
+ hidden_states = residual + hidden_states
596
+
597
+ outputs = (hidden_states, )
598
+
599
+ if output_attentions:
600
+ outputs += (attn_weights, )
601
+
602
+ return outputs
603
+
604
+
605
+class SiglipPreTrainedModel(PreTrainedModel):
606
+ config_class = SiglipVisionConfig
607
+ base_model_prefix = "siglip"
608
+ supports_gradient_checkpointing = True
609
+
610
+ def _init_weights(self, module):
611
+ """Initialize the weights"""
612
+
613
+ if isinstance(module, SiglipVisionEmbeddings):
614
+ width = self.config.hidden_size
615
+ nn.init.normal_(module.position_embedding.weight,
616
+ std=1 / np.sqrt(width))
617
+ elif isinstance(module, nn.Embedding):
618
+ default_flax_embed_init(module.weight)
619
+ elif isinstance(module, SiglipAttention):
620
+ nn.init.normal_(module.q_proj.weight)
621
+ nn.init.normal_(module.k_proj.weight)
622
+ nn.init.normal_(module.v_proj.weight)
623
+ nn.init.normal_(module.out_proj.weight)
624
+ nn.init.zeros_(module.q_proj.bias)
625
+ nn.init.zeros_(module.k_proj.bias)
626
+ nn.init.zeros_(module.v_proj.bias)
627
+ nn.init.zeros_(module.out_proj.bias)
628
+ elif isinstance(module, SiglipMLP):
629
+ nn.init.normal_(module.fc1.weight)
630
+ nn.init.normal_(module.fc2.weight)
631
+ nn.init.normal_(module.fc1.bias, std=1e-6)
632
+ nn.init.normal_(module.fc2.bias, std=1e-6)
633
+ elif isinstance(module, (nn.Linear, nn.Conv2d)):
634
+ lecun_normal_(module.weight)
635
+ if module.bias is not None:
636
+ nn.init.zeros_(module.bias)
637
+ elif isinstance(module, nn.LayerNorm):
638
+ module.bias.data.zero_()
639
+ module.weight.data.fill_(1.0)
640
+
641
+
642
+# Copied from transformers.models.clip.modeling_clip.CLIPEncoder
643
+# with CLIP->Siglip
644
+class SiglipEncoder(nn.Module):
645
+
646
+ def __init__(self, config: SiglipVisionConfig):
647
+ super().__init__()
648
+ self.config = config
649
+ self.layers = nn.ModuleList([
650
+ SiglipEncoderLayer(config) for _ in range(config.num_hidden_layers)
651
+ ])
652
+ self.gradient_checkpointing = False
653
+
654
+ # Ignore copy
655
+ def forward(
656
+ self,
657
+ inputs_embeds,
658
+ attention_mask: Optional[torch.Tensor] = None,
659
+ output_attentions: Optional[bool] = None,
660
+ output_hidden_states: Optional[bool] = None,
661
+ return_dict: Optional[bool] = None,
662
+ ) -> Union[Tuple, BaseModelOutput]:
663
+ output_attentions = output_attentions if output_attentions is not None \
664
+ else self.config.output_attentions
665
+ output_hidden_states = (output_hidden_states
666
+ if output_hidden_states is not None else
667
+ self.config.output_hidden_states)
668
+ return_dict = return_dict if return_dict is not None \
669
+ else self.config.use_return_dict
670
+
671
+ encoder_states = () if output_hidden_states else None
672
+ all_attentions = () if output_attentions else None
673
+
674
+ hidden_states = inputs_embeds
675
+ for encoder_layer in self.layers:
676
+ if output_hidden_states:
677
+ encoder_states = encoder_states + (hidden_states, )
678
+ if self.gradient_checkpointing and self.training:
679
+ layer_outputs = self._gradient_checkpointing_func(
680
+ encoder_layer.__call__,
681
+ hidden_states,
682
+ attention_mask,
683
+ output_attentions,
684
+ )
685
+ else:
686
+ layer_outputs = encoder_layer(
687
+ hidden_states,
688
+ attention_mask,
689
+ output_attentions=output_attentions,
690
+ )
691
+
692
+ hidden_states = layer_outputs[0]
693
+
694
+ if output_attentions:
695
+ all_attentions = all_attentions + (layer_outputs[1], )
696
+
697
+ if output_hidden_states:
698
+ encoder_states = encoder_states + (hidden_states, )
699
+
700
+ if not return_dict:
701
+ return tuple(
702
+ v for v in [hidden_states, encoder_states, all_attentions]
703
+ if v is not None)
704
+ return BaseModelOutput(last_hidden_state=hidden_states,
705
+ hidden_states=encoder_states,
706
+ attentions=all_attentions)
707
+
708
+
709
+class SiglipVisionTransformer(SiglipPreTrainedModel):
710
+ config_class = SiglipVisionConfig
711
+ main_input_name = "pixel_values"
712
+ _supports_flash_attn_2 = True
713
+
714
+ def __init__(self, config: SiglipVisionConfig):
715
+ super().__init__(config)
716
+ self.config = config
717
+ embed_dim = config.hidden_size
718
+
719
+ self.embeddings = SiglipVisionEmbeddings(config)
720
+ self.encoder = SiglipEncoder(config)
721
+ self.post_layernorm = nn.LayerNorm(embed_dim,
722
+ eps=config.layer_norm_eps)
723
+ self._use_flash_attention_2 = (
724
+ config._attn_implementation == "flash_attention_2")
725
+
726
+ # Initialize weights and apply final processing
727
+ self.post_init()
728
+
729
+ def get_input_embeddings(self) -> nn.Module:
730
+ return self.embeddings.patch_embedding
731
+
732
+ @replace_return_docstrings(output_type=BaseModelOutputWithPooling,
733
+ config_class=SiglipVisionConfig)
734
+ def forward(
735
+ self,
736
+ pixel_values,
737
+ patch_attention_mask: Optional[torch.BoolTensor] = None,
738
+ tgt_sizes: Optional[torch.IntTensor] = None,
739
+ output_attentions: Optional[bool] = None,
740
+ output_hidden_states: Optional[bool] = None,
741
+ return_dict: Optional[bool] = None,
742
+ ) -> Union[Tuple, BaseModelOutputWithPooling]:
743
+ r"""
744
+ Returns:
745
+ """
746
+ output_attentions = output_attentions if output_attentions is not None \
747
+ else self.config.output_attentions
748
+ output_hidden_states = (output_hidden_states
749
+ if output_hidden_states is not None else
750
+ self.config.output_hidden_states)
751
+ return_dict = return_dict if return_dict is not None \
752
+ else self.config.use_return_dict
753
+
754
+ batch_size = pixel_values.size(0)
755
+ if patch_attention_mask is None:
756
+ patch_attention_mask = torch.ones(
757
+ size=(
758
+ batch_size,
759
+ pixel_values.size(2) // self.config.patch_size,
760
+ pixel_values.size(3) // self.config.patch_size,
761
+ ),
762
+ dtype=torch.bool,
763
+ device=pixel_values.device,
764
+ )
765
+
766
+ hidden_states = self.embeddings(
767
+ pixel_values=pixel_values,
768
+ patch_attention_mask=patch_attention_mask,
769
+ tgt_sizes=tgt_sizes)
770
+
771
+ patch_attention_mask = patch_attention_mask.view(batch_size, -1)
772
+ # The call to `_upad_input` in `_flash_attention_forward` is expensive
773
+ # So when the `patch_attention_mask` is full of 1s
774
+ # (i.e. attending to the whole sequence),
775
+ # avoiding passing the attention_mask,
776
+ # which is equivalent to attending to the full sequence
777
+ if not torch.any(~patch_attention_mask):
778
+ attention_mask = None
779
+ else:
780
+ attention_mask = (_prepare_4d_attention_mask(
781
+ patch_attention_mask, hidden_states.dtype)
782
+ if not self._use_flash_attention_2 else
783
+ patch_attention_mask)
784
+
785
+ encoder_outputs = self.encoder(
786
+ inputs_embeds=hidden_states,
787
+ attention_mask=attention_mask,
788
+ output_attentions=output_attentions,
789
+ output_hidden_states=output_hidden_states,
790
+ return_dict=return_dict,
791
+ )
792
+
793
+ last_hidden_state = encoder_outputs[0]
794
+ last_hidden_state = self.post_layernorm(last_hidden_state)
795
+
796
+ if not return_dict:
797
+ return (last_hidden_state, None) + encoder_outputs[1:]
798
+
799
+ return BaseModelOutputWithPooling(
800
+ last_hidden_state=last_hidden_state,
801
+ pooler_output=None,
802
+ hidden_states=encoder_outputs.hidden_states,
803
+ attentions=encoder_outputs.attentions,
804
+ )
vllm/model_executor/models/qwen2.py
+1
-1
@@ -342,7 +342,7 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA):
342
intermediate_tensors: Optional[IntermediateTensors] = None,
343
) -> torch.Tensor:
344
hidden_states = self.model(input_ids, positions, kv_caches,
345
- attn_metadata)
345
+ attn_metadata, intermediate_tensors)
346
return hidden_states
347
348
def compute_logits(self, hidden_states: torch.Tensor,