@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2f4e108f75
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Bugfix] Clean up MiniCPM-V (#6939)

Co-authored-by: hezhihui <hzh7269@modelbest.cn> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>

Alphi committed Jul 31, 2024 at 22:39 UTC 2f4e108f75c817bf2f323e306db590e13d2863f6
6 files changed +975 -94
docs/source/models/supported_models.rst
+5 -1
@@ -222,9 +222,13 @@ Vision Language Models
222 -
223 * - :code:`MiniCPM-V`
224 - MiniCPM-V
225 - - :code:`openbmb/MiniCPM-V-2`, :code:`openbmb/MiniCPM-Llama3-V-2_5`, etc.
225 + - :code:`openbmb/MiniCPM-V-2` (see note), :code:`openbmb/MiniCPM-Llama3-V-2_5`, etc.
226 -
227
228 +.. note::
229 + For :code:`openbmb/MiniCPM-V-2`, the official repo doesn't work yet, so we need to use a fork (:code:`HwwwH/MiniCPM-V-2`) for now.
230 + For more details, please see: https://github.com/vllm-project/vllm/pull/4087#issuecomment-2250397630
231 +
232 ----
233
234 If your model uses one of the above model architectures, you can seamlessly run your model with vLLM.
vllm/model_executor/models/llama.py
+1 -3
@@ -418,11 +418,9 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA):
418 kv_caches: List[torch.Tensor],
419 attn_metadata: AttentionMetadata,
420 intermediate_tensors: Optional[IntermediateTensors] = None,
421 - input_embeds: Optional[torch.Tensor] = None
421 ) -> Union[torch.Tensor, IntermediateTensors]:
422 model_output = self.model(input_ids, positions, kv_caches,
424 - attn_metadata, intermediate_tensors,
425 - input_embeds)
423 + attn_metadata, intermediate_tensors)
424 return model_output
425
426 def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/minicpm.py
+2 -2
@@ -370,6 +370,7 @@ class MiniCPMModel(nn.Module):
370 positions: torch.Tensor,
371 kv_caches: List[torch.Tensor],
372 attn_metadata: AttentionMetadata,
373 + intermediate_tensors: Optional[IntermediateTensors] = None,
374 inputs_embeds: Optional[torch.Tensor] = None,
375 ) -> torch.Tensor:
376 if inputs_embeds is not None:
@@ -463,11 +464,10 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA):
464 positions: torch.Tensor,
465 kv_caches: List[torch.Tensor],
466 attn_metadata: AttentionMetadata,
466 - input_embeds: Optional[torch.Tensor] = None,
467 intermediate_tensors: Optional[IntermediateTensors] = None,
468 ) -> torch.Tensor:
469 hidden_states = self.model(input_ids, positions, kv_caches,
470 - attn_metadata, input_embeds)
470 + attn_metadata, intermediate_tensors)
471 return hidden_states
472
473 def compute_logits(self, hidden_states: torch.Tensor,
vllm/model_executor/models/minicpmv.py
+162 -87
@@ -20,32 +20,34 @@
20 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
21 # See the License for the specific language governing permissions and
22 # limitations under the License.
23 -"""Inference-only MiniCPM-V-2 model compatible with HuggingFace weights."""
23 +"""Inference-only MiniCPM-V model compatible with HuggingFace weights."""
24 import math
25 import re
26 from functools import partial
27 -from typing import Iterable, List, Optional, Tuple
27 +from typing import Dict, Iterable, List, Optional, Tuple, Union
28
29 import numpy as np
30 import torch
31 import torch.nn.functional as F
32 +import torch.types
33 from PIL import Image
34 from torch import nn
35 from torch.nn.init import trunc_normal_
36 from transformers.configuration_utils import PretrainedConfig
36 -from transformers.models.idefics2.modeling_idefics2 import (
37 - Idefics2VisionTransformer)
37
38 from vllm.attention import AttentionMetadata
39 from vllm.config import CacheConfig, MultiModalConfig
40 from vllm.inputs import INPUT_REGISTRY, InputContext, LLMInputs
41 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
42 from vllm.model_executor.layers.quantization.base_config import (
43 QuantizationConfig)
44 from vllm.model_executor.layers.sampler import Sampler
45 +from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
46 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
47 from vllm.model_executor.models.interfaces import SupportsVision
47 -from vllm.model_executor.models.llama import LlamaForCausalLM
48 -from vllm.model_executor.models.minicpm import MiniCPMForCausalLM
48 +from vllm.model_executor.models.llama import LlamaModel
49 +from vllm.model_executor.models.minicpm import MiniCPMModel
50 +from vllm.model_executor.models.qwen2 import Qwen2Model
51 from vllm.model_executor.sampling_metadata import SamplingMetadata
52 from vllm.multimodal import MULTIMODAL_REGISTRY
53 from vllm.multimodal.image import (cached_get_image_processor,
@@ -53,12 +55,12 @@ from vllm.multimodal.image import (cached_get_image_processor,
55 from vllm.sequence import IntermediateTensors, SamplerOutput, SequenceData
56
57 _KEYS_TO_MODIFY_MAPPING = {
56 - "language_model.lm_head": "lm_head",
57 - "language_model.model": "language_model",
58 + "llm.lm_head": "lm_head",
59 + "llm.model": "llm",
60 }
61
62
61 -def get_abs_pos(abs_pos, tgt_size):
63 +def get_abs_pos(abs_pos: torch.Tensor, tgt_size: torch.Tensor):
64 # abs_pos: L, C
65 # tgt_size: (H, W)
66 # return: M, C
@@ -75,10 +77,10 @@ def get_abs_pos(abs_pos, tgt_size):
77
78
79 # https://github.com/facebookresearch/mae/blob/efb2a8062c206524e35e47d04501ed4f544c0ae8/util/pos_embed.py#L20
78 -def get_2d_sincos_pos_embed(embed_dim,
79 - grid_size,
80 - cls_token=False,
81 - version=2.0):
80 +def get_2d_sincos_pos_embed(embed_dim: int,
81 + grid_size: Union[int, Tuple[int, int]],
82 + cls_token: bool = False,
83 + version: Tuple[int, int] = (2, 0)):
84 """
85 grid_size: int of the grid height and width
86 return:
@@ -95,7 +97,7 @@ def get_2d_sincos_pos_embed(embed_dim,
97 grid = np.meshgrid(grid_w, grid_h) # here w goes first
98 grid = np.stack(grid, axis=0)
99
98 - if version == 2.0:
100 + if version == (2, 0):
101 grid = grid.reshape([2, 1, grid_h_size, grid_w_size])
102 pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid, version)
103 if cls_token:
@@ -106,7 +108,9 @@ def get_2d_sincos_pos_embed(embed_dim,
108 return pos_embed
109
110
109 -def get_2d_sincos_pos_embed_from_grid(embed_dim, grid, version=2.0):
111 +def get_2d_sincos_pos_embed_from_grid(embed_dim: int,
112 + grid: Union[int, Tuple[int, int]],
113 + version: Tuple[int, int] = (2, 0)):
114 assert embed_dim % 2 == 0
115
116 # use half of dimensions to encode grid_h
@@ -115,14 +119,16 @@ def get_2d_sincos_pos_embed_from_grid(embed_dim, grid, version=2.0):
119 emb_w = get_1d_sincos_pos_embed_from_grid(
120 embed_dim // 2, grid[1], version) # (H*W, D/2) or (H, W, D/2)
121
118 - if version == 2.0:
122 + if version == (2, 0):
123 emb = np.concatenate([emb_h, emb_w], axis=1) # (H*W, D)
124 else:
125 emb = np.concatenate([emb_h, emb_w], axis=-1) # (H, W, D)
126 return emb
127
128
125 -def get_1d_sincos_pos_embed_from_grid(embed_dim, pos, version=2.0):
129 +def get_1d_sincos_pos_embed_from_grid(embed_dim: int,
130 + pos: int,
131 + version: Tuple[int, int] = (2, 0)):
132 """
133 embed_dim: output dimension for each position
134 pos: a list of positions to be encoded: size (M,) / (H, W)
@@ -133,7 +139,7 @@ def get_1d_sincos_pos_embed_from_grid(embed_dim, pos, version=2.0):
139 omega /= embed_dim / 2.
140 omega = 1. / 10000**omega # (D/2,)
141
136 - if version == 2.0:
142 + if version == (2, 0):
143 pos = pos.reshape(-1) # (M,)
144 out = np.einsum('m,d->md', pos, omega) # (M, D/2), outer product
145 emb_sin = np.sin(out) # (M, D/2)
@@ -158,19 +164,19 @@ class Resampler(nn.Module):
164 default_norm_layer = partial(nn.LayerNorm, eps=1e-6)
165
166 def __init__(self,
161 - num_queries,
162 - grid_size,
163 - embed_dim,
164 - num_heads,
165 - kv_dim=None,
166 - norm_layer=default_norm_layer,
167 - adaptive=False,
168 - max_size=(70, 70),
169 - version=2.0):
167 + num_queries: int,
168 + grid_size: int,
169 + embed_dim: int,
170 + num_heads: int,
171 + kv_dim: Optional[int] = None,
172 + norm_layer: nn.Module = default_norm_layer,
173 + adaptive: bool = False,
174 + max_size: Tuple[int, int] = (70, 70),
175 + version: Tuple[int, int] = (2, 0)):
176 super().__init__()
177
178 self.version = version
173 - if self.version == 2.0:
179 + if self.version == (2, 0):
180 self.num_queries = grid_size**2
181 else:
182 self.num_queries = num_queries
@@ -195,7 +201,7 @@ class Resampler(nn.Module):
201 self.proj = nn.Parameter(
202 (embed_dim**-0.5) * torch.randn(embed_dim, embed_dim))
203
198 - if self.version == 2.0:
204 + if self.version == (2, 0):
205 self.pos_embed = nn.Parameter(
206 torch.from_numpy(
207 get_2d_sincos_pos_embed(
@@ -206,14 +212,17 @@ class Resampler(nn.Module):
212
213 self.apply(self._init_weights)
214
209 - def _set_2d_pos_cache(self, max_size, device='cpu'):
215 + def _set_2d_pos_cache(self,
216 + max_size: Tuple[int, int],
217 + device: torch.types.Device = 'cpu'):
218 pos_embed = torch.from_numpy(
219 get_2d_sincos_pos_embed(self.embed_dim,
220 max_size,
221 version=self.version)).float().to(device)
222 self.register_buffer("pos_embed", pos_embed, persistent=False)
223
216 - def _adjust_pos_cache(self, tgt_sizes, device):
224 + def _adjust_pos_cache(self, tgt_sizes: torch.Tensor,
225 + device: torch.types.Device):
226 max_h = torch.max(tgt_sizes[:, 0])
227 max_w = torch.max(tgt_sizes[:, 1])
228 if max_h > self.max_size[0] or max_w > self.max_size[1]:
@@ -223,7 +232,7 @@ class Resampler(nn.Module):
232 ]
233 self._set_2d_pos_cache(self.max_size, device)
234
226 - def _init_weights(self, m):
235 + def _init_weights(self, m: nn.Module):
236 if isinstance(m, nn.Linear):
237 trunc_normal_(m.weight, std=.02)
238 if isinstance(m, nn.Linear) and m.bias is not None:
@@ -232,7 +241,9 @@ class Resampler(nn.Module):
241 nn.init.constant_(m.bias, 0)
242 nn.init.constant_(m.weight, 1.0)
243
235 - def forward_2_5(self, x, tgt_sizes=None):
244 + def forward_2_5(self,
245 + x: torch.Tensor,
246 + tgt_sizes: Optional[torch.Tensor] = None):
247 assert x.shape[0] == tgt_sizes.shape[0]
248 bs = x.shape[0]
249
@@ -278,7 +289,10 @@ class Resampler(nn.Module):
289 x = x @ self.proj
290 return x
291
281 - def forward_2(self, x, tgt_sizes=None, attn_mask=None):
292 + def forward_2(self,
293 + x: torch.Tensor,
294 + tgt_sizes: Optional[torch.Tensor] = None,
295 + attn_mask: Optional[torch.Tensor] = None):
296 if self.adaptive:
297 pos_embed = torch.Tensor(
298 get_2d_sincos_pos_embed(self.embed_dim,
@@ -302,8 +316,11 @@ class Resampler(nn.Module):
316 x = x @ self.proj
317 return x
318
305 - def forward(self, x, tgt_sizes=None, attn_mask=None):
306 - if self.version == 2.0:
319 + def forward(self,
320 + x: torch.Tensor,
321 + tgt_sizes: Optional[torch.Tensor] = None,
322 + attn_mask: Optional[torch.Tensor] = None):
323 + if self.version == (2, 0):
324 return self.forward_2(x, tgt_sizes=tgt_sizes, attn_mask=attn_mask)
325 else:
326 return self.forward_2_5(x, tgt_sizes=tgt_sizes)
@@ -322,7 +339,7 @@ def dummy_seq_data_for_minicpmv(seq_len: int):
339 return SequenceData(token_ids)
340
341
325 -def dummy_image_for_minicpmv(hf_config):
342 +def dummy_image_for_minicpmv(hf_config: PretrainedConfig):
343 width = height = hf_config.image_size
344 image = Image.new("RGB", (width, height), color=0)
345 return {"image": image}
@@ -381,7 +398,7 @@ class MiniCPMV(nn.Module, SupportsVision):
398
399 def __init__(
400 self,
384 - config,
401 + config: PretrainedConfig,
402 multimodal_config: MultiModalConfig,
403 cache_config: Optional[CacheConfig] = None,
404 quant_config: Optional[QuantizationConfig] = None,
@@ -390,30 +407,48 @@ class MiniCPMV(nn.Module, SupportsVision):
407 self.config = config
408 self.multimodal_config = multimodal_config
409
393 - self.version = float(self.config.version)
410 + if not hasattr(self.config, "version"):
411 + if self.config.hidden_size == 2304 and self.config.query_num == 64:
412 + self.version = (2, 0)
413 + else:
414 + self.version = (2, 5)
415 + else:
416 + self.version = str(self.config.version).split(".")
417 + self.version = tuple([int(x) for x in self.version])
418 self.llm = self.init_llm(config, cache_config, quant_config)
419 self.vpm = self.init_vision_module()
420 param_dtype = torch.get_default_dtype()
421 self.vpm.to(dtype=param_dtype)
398 - self.vision_dim = self.vpm.embed_dim if self.version == 2.0 \
422 + self.vision_dim = self.vpm.embed_dim if self.version == (2, 0) \
423 else self.vpm.embeddings.embed_dim
400 - self.embed_dim = self.llm.config.hidden_size
424 + self.embed_dim = self.config.hidden_size
425 self.resampler = self.init_resampler(self.embed_dim, self.vision_dim)
426 self.resampler.to(device="cuda", dtype=param_dtype)
427 + self.lm_head = ParallelLMHead(config.vocab_size,
428 + config.hidden_size,
429 + quant_config=quant_config)
430 + self.logits_processor = LogitsProcessor(config.vocab_size)
431 self.sampler = Sampler()
432
405 - def init_llm(self, config, cache_config, quant_config):
406 - if self.version == 2.0:
407 - return MiniCPMForCausalLM(config,
408 - cache_config=cache_config,
409 - quant_config=quant_config)
433 + def init_llm(self,
434 + config: PretrainedConfig,
435 + cache_config: Optional[CacheConfig] = None,
436 + quant_config: Optional[QuantizationConfig] = None):
437 + if self.version == (2, 0):
438 + return MiniCPMModel(config,
439 + cache_config=cache_config,
440 + quant_config=quant_config)
441 + elif self.version == (2, 5):
442 + return LlamaModel(config,
443 + cache_config=cache_config,
444 + quant_config=quant_config)
445 else:
411 - return LlamaForCausalLM(config,
412 - cache_config=cache_config,
413 - quant_config=quant_config)
446 + return Qwen2Model(config,
447 + cache_config=cache_config,
448 + quant_config=quant_config)
449
450 def init_vision_module(self):
416 - if self.version == 2.0:
451 + if self.version == (2, 0):
452 try:
453 import timm
454 except ImportError:
@@ -433,16 +468,30 @@ class MiniCPMV(nn.Module, SupportsVision):
468
469 if self.config.drop_vision_last_layer:
470 model.blocks = model.blocks[:-1]
436 - else:
471 + elif self.version == (2, 5):
472 + from transformers.models.idefics2.modeling_idefics2 import (
473 + Idefics2VisionTransformer)
474 model = Idefics2VisionTransformer(self.config.vision_config)
475 if self.config.drop_vision_last_layer:
476 model.encoder.layers = model.encoder.layers[:-1]
477 + else:
478 + from vllm.model_executor.models.na_vit import (
479 + SiglipVisionTransformer)
480 + if self.config._attn_implementation == 'flash_attention_2':
481 + self.config.vision_config._attn_implementation \
482 + = 'flash_attention_2'
483 + else:
484 + # not support sdpa
485 + self.config.vision_config._attn_implementation = 'eager'
486 + model = SiglipVisionTransformer(self.config.vision_config)
487 + if self.config.drop_vision_last_layer:
488 + model.encoder.layers = model.encoder.layers[:-1]
489 return model
490
442 - def init_resampler(self, embed_dim, vision_dim):
491 + def init_resampler(self, embed_dim: int, vision_dim: int):
492 default_dtype = torch.get_default_dtype()
493 torch.set_default_dtype(torch.float16)
445 - if self.version == 2.0:
494 + if self.version == (2, 0):
495 resampler = Resampler(grid_size=int(
496 math.sqrt(self.config.query_num)),
497 num_queries=None,
@@ -463,11 +512,11 @@ class MiniCPMV(nn.Module, SupportsVision):
512 return resampler
513
514 def get_vision_embedding(self,
466 - pixel_values,
467 - patch_attn_mask=None,
468 - tgt_sizes=None,
469 - version=2.0):
470 - if version == 2.0:
515 + pixel_values: List[List[torch.Tensor]],
516 + patch_attn_mask: Optional[torch.Tensor] = None,
517 + tgt_sizes: Optional[torch.Tensor] = None,
518 + version: Tuple[int, int] = (2, 0)):
519 + if version == (2, 0):
520 res = []
521 dtype = self.vpm.pos_embed.data.dtype
522 for pixel_value in pixel_values:
@@ -484,21 +533,32 @@ class MiniCPMV(nn.Module, SupportsVision):
533 num_prefix_tokens:]
534 res.append(self.resampler(vision_embedding, tgt_size))
535 return torch.vstack(res)
487 - else:
536 + elif version == (2, 5):
537 vision_embedding = self.vpm(
538 pixel_values.type(dtype),
539 patch_attention_mask=patch_attn_mask).last_hidden_state
540 vision_embedding = self.resampler(vision_embedding, tgt_sizes)
541 + else:
542 + vision_embedding = self.vpm(pixel_values.type(dtype),
543 + patch_attention_mask=patch_attn_mask,
544 + tgt_sizes=tgt_sizes).last_hidden_state
545
493 - def get_image_bounds(self, input_ids):
546 + def get_image_bounds(self, input_ids: torch.Tensor):
547 tokenizer = cached_get_tokenizer(self.config._name_or_path,
548 trust_remote_code=True)
496 - im_start_token_id = tokenizer.im_start_id
497 - im_end_token_id = tokenizer.im_end_id
498 - image_start_tokens = torch.where(input_ids == im_start_token_id)[0]
549 + if not hasattr(tokenizer, "slice_start_id"):
550 + start_cond = input_ids == tokenizer.im_start_id
551 + end_cond = input_ids == tokenizer.im_end_id
552 + else:
553 + start_cond = (input_ids == tokenizer.im_start_id) | (
554 + input_ids == tokenizer.slice_start_id)
555 + end_cond = (input_ids == tokenizer.im_end_id) | (
556 + input_ids == tokenizer.slice_end_id)
557 +
558 + image_start_tokens = torch.where(start_cond)[0]
559 image_start_tokens += 1
500 - image_end_tokens = torch.where(input_ids == im_end_token_id)[0]
501 - valid_image_nums = min(len(image_start_tokens), len(image_end_tokens))
560 + image_end_tokens = torch.where(end_cond)[0]
561 + valid_image_nums = max(len(image_start_tokens), len(image_end_tokens))
562 if valid_image_nums == 0:
563 return []
564 image_bound = torch.hstack([
@@ -508,12 +568,14 @@ class MiniCPMV(nn.Module, SupportsVision):
568
569 return image_bound
570
511 - def get_vision_hidden_states(self, data):
571 + def get_vision_hidden_states(self, data: Dict[str,
572 + Union[List[torch.Tensor],
573 + torch.Tensor]]):
574 if "vision_hidden_states" not in data:
575 pixel_values = data["pixel_values"]
576 tgt_sizes = data["tgt_sizes"]
577 vision_hidden_states = []
516 - if self.version == 2.0:
578 + if self.version == (2, 0):
579 if pixel_values is not None and len(pixel_values) > 0:
580 vision_hidden_states = self.get_vision_embedding(
581 pixel_values)
@@ -534,17 +596,26 @@ class MiniCPMV(nn.Module, SupportsVision):
596 B, L, _ = all_pixel_values.shape
597 all_pixel_values = all_pixel_values.permute(
598 0, 2, 1).reshape(B, 3, -1, L)
537 -
599 patch_attn_mask = torch.zeros((B, 1, max_patches),
600 dtype=torch.bool,
601 device=device)
541 - for i in range(B):
542 - patch_attn_mask[i, :tgt_sizes[i][0] *
543 - tgt_sizes[i][1]] = True
602 + if self.version == (2, 5):
603 + for i in range(B):
604 + patch_attn_mask[i, :tgt_sizes[i][0] *
605 + tgt_sizes[i][1]] = True
606 + vision_embedding = self.vpm(
607 + all_pixel_values.type(dtype),
608 + patch_attention_mask=patch_attn_mask
609 + ).last_hidden_state
610 + else:
611 + for i in range(B):
612 + patch_attn_mask[i, 0, :tgt_sizes[i][0] *
613 + tgt_sizes[i][1]] = True
614 + vision_embedding = self.vpm(
615 + all_pixel_values.type(dtype),
616 + patch_attention_mask=patch_attn_mask,
617 + tgt_sizes=tgt_sizes).last_hidden_state
618
545 - vision_embedding = self.vpm(
546 - all_pixel_values.type(dtype),
547 - patch_attention_mask=patch_attn_mask).last_hidden_state
619 vision_hidden_states = self.resampler(
620 vision_embedding, tgt_sizes)
621
@@ -556,7 +627,8 @@ class MiniCPMV(nn.Module, SupportsVision):
627
628 return vision_hidden_states
629
559 - def get_embedding(self, data):
630 + def get_embedding(self, data: Dict[str, Union[List[torch.Tensor],
631 + torch.Tensor]]):
632 input_ids = data["input_ids"]
633
634 vision_hidden_states = self.get_vision_hidden_states(data)
@@ -565,11 +637,11 @@ class MiniCPMV(nn.Module, SupportsVision):
637 else:
638 image_bounds = []
639
568 - if hasattr(self.llm.config, 'scale_emb'):
569 - vlm_embedding = self.llm.model.embed_tokens(
570 - input_ids) * self.llm.config.scale_emb
640 + if hasattr(self.config, 'scale_emb'):
641 + vlm_embedding = self.llm.embed_tokens(
642 + input_ids) * self.config.scale_emb
643 else:
572 - vlm_embedding = self.llm.model.embed_tokens(input_ids)
644 + vlm_embedding = self.llm.embed_tokens(input_ids)
645 vision_hidden_states = [
646 i.type(vlm_embedding.dtype) if isinstance(i, torch.Tensor) else i
647 for i in vision_hidden_states
@@ -587,7 +659,9 @@ class MiniCPMV(nn.Module, SupportsVision):
659 vision_hidden_states.view(-1, vision_hidden_states.shape[-1]))
660 return vlm_embedding, vision_hidden_states
661
590 - def process_multimodal_inputs(self, inputs):
662 + def process_multimodal_inputs(self, inputs: Dict[str,
663 + Union[List[torch.Tensor],
664 + torch.Tensor]]):
665 pixel_values = []
666 tgt_sizes = []
667 for b in range(len(inputs["pixel_values"])):
@@ -613,7 +687,6 @@ class MiniCPMV(nn.Module, SupportsVision):
687 "input_ids": input_ids,
688 "tgt_sizes": kwargs.pop("tgt_sizes", None),
689 }
616 -
690 inputs = self.process_multimodal_inputs(inputs)
691
692 vlm_embeddings, vision_hidden_states = self.get_embedding(inputs)
@@ -623,19 +696,21 @@ class MiniCPMV(nn.Module, SupportsVision):
696 kv_caches=kv_caches,
697 attn_metadata=attn_metadata,
698 intermediate_tensors=intermediate_tensors,
626 - input_embeds=vlm_embeddings)
699 + inputs_embeds=vlm_embeddings)
700 return output
701
702 def compute_logits(self, hidden_states: torch.Tensor,
703 sampling_metadata: SamplingMetadata) -> torch.Tensor:
631 - return self.llm.compute_logits(hidden_states, sampling_metadata)
704 + logits = self.logits_processor(self.lm_head, hidden_states,
705 + sampling_metadata)
706 + return logits
707
708 def sample(
709 self,
710 logits: torch.Tensor,
711 sampling_metadata: SamplingMetadata,
712 ) -> Optional[SamplerOutput]:
638 - next_tokens = self.llm.sample(logits, sampling_metadata)
713 + next_tokens = self.sampler(logits, sampling_metadata)
714 return next_tokens
715
716 def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
@@ -649,9 +724,9 @@ class MiniCPMV(nn.Module, SupportsVision):
724 ]
725 params_dict = dict(self.named_parameters())
726 for name, loaded_weight in weights:
652 - # for key_to_modify, new_key in _KEYS_TO_MODIFY_MAPPING.items():
653 - # if key_to_modify in name:
654 - # name = name.replace(key_to_modify, new_key)
727 + for key_to_modify, new_key in _KEYS_TO_MODIFY_MAPPING.items():
728 + if key_to_modify in name:
729 + name = name.replace(key_to_modify, new_key)
730 if "rotary_emb.inv_freq" in name:
731 continue
732 if ("rotary_emb.cos_cached" in name
vllm/model_executor/models/na_vit.py new
+804
@@ -0,0 +1,804 @@
1 +import logging
2 +import math
3 +import os
4 +import warnings
5 +from typing import Optional, Tuple, Union
6 +
7 +import numpy as np
8 +import torch
9 +import torch.nn.functional as F
10 +from torch import nn
11 +from torch.nn.init import _calculate_fan_in_and_fan_out
12 +from transformers.activations import ACT2FN
13 +from transformers.configuration_utils import PretrainedConfig
14 +from transformers.modeling_attn_mask_utils import _prepare_4d_attention_mask
15 +from transformers.modeling_outputs import (BaseModelOutput,
16 + BaseModelOutputWithPooling)
17 +from transformers.modeling_utils import PreTrainedModel
18 +from transformers.utils import (ModelOutput, is_flash_attn_2_available,
19 + replace_return_docstrings)
20 +
21 +logger = logging.getLogger("vllm")
22 +
23 +
24 +# For Siglip: copied from
25 +# HuggingFaceM4/siglip-so400m-14-980-flash-attn2-navit and add tgt_sizes
26 +# Remove hints as there's little possibility to change these code.
27 +class SiglipVisionConfig(PretrainedConfig):
28 +
29 + model_type = "siglip_vision_model"
30 +
31 + def __init__(
32 + self,
33 + hidden_size=768,
34 + intermediate_size=3072,
35 + num_hidden_layers=12,
36 + num_attention_heads=12,
37 + num_channels=3,
38 + image_size=224,
39 + patch_size=16,
40 + hidden_act="gelu_pytorch_tanh",
41 + layer_norm_eps=1e-6,
42 + attention_dropout=0.0,
43 + **kwargs,
44 + ):
45 + super().__init__(**kwargs)
46 +
47 + self.hidden_size = hidden_size
48 + self.intermediate_size = intermediate_size
49 + self.num_hidden_layers = num_hidden_layers
50 + self.num_attention_heads = num_attention_heads
51 + self.num_channels = num_channels
52 + self.patch_size = patch_size
53 + self.image_size = image_size
54 + self.attention_dropout = attention_dropout
55 + self.layer_norm_eps = layer_norm_eps
56 + self.hidden_act = hidden_act
57 +
58 + @classmethod
59 + def from_pretrained(cls, pretrained_model_name_or_path: Union[str,
60 + os.PathLike],
61 + **kwargs) -> "PretrainedConfig":
62 + cls._set_token_in_kwargs(kwargs)
63 +
64 + config_dict, kwargs = cls.get_config_dict(
65 + pretrained_model_name_or_path, **kwargs)
66 +
67 + # get the vision config dict if we are loading from SiglipConfig
68 + if config_dict.get("model_type") == "siglip":
69 + config_dict = config_dict["vision_config"]
70 +
71 + if "model_type" in config_dict and hasattr(
72 + cls,
73 + "model_type") and config_dict["model_type"] != cls.model_type:
74 + logger.warning(
75 + "You are using a model of type %s to "
76 + "instantiate a model of type %s. "
77 + "This is not supported for all configurations"
78 + "of models and can yield errors.", config_dict['model_type'],
79 + cls.model_type)
80 +
81 + return cls.from_dict(config_dict, **kwargs)
82 +
83 +
84 +_CHECKPOINT_FOR_DOC = "google/siglip-base-patch16-224"
85 +
86 +SIGLIP_PRETRAINED_MODEL_ARCHIVE_LIST = [
87 + "google/siglip-base-patch16-224",
88 + # See all SigLIP models at https://huggingface.co/models?filter=siglip
89 +]
90 +
91 +if is_flash_attn_2_available():
92 + from flash_attn import flash_attn_func, flash_attn_varlen_func
93 + from flash_attn.bert_padding import pad_input # noqa
94 + from flash_attn.bert_padding import index_first_axis, unpad_input
95 +
96 +
97 +# Copied from transformers.models.llama.modeling_llama._get_unpad_data
98 +def _get_unpad_data(attention_mask):
99 + seqlens_in_batch = attention_mask.sum(dim=-1, dtype=torch.int32)
100 + indices = torch.nonzero(attention_mask.flatten(), as_tuple=False).flatten()
101 + max_seqlen_in_batch = seqlens_in_batch.max().item()
102 + cu_seqlens = F.pad(
103 + torch.cumsum(seqlens_in_batch, dim=0, dtype=torch.torch.int32), (1, 0))
104 + return (
105 + indices,
106 + cu_seqlens,
107 + max_seqlen_in_batch,
108 + )
109 +
110 +
111 +def _trunc_normal_(tensor, mean, std, a, b):
112 +
113 + def norm_cdf(x):
114 + # Computes standard normal cumulative distribution function
115 + return (1.0 + math.erf(x / math.sqrt(2.0))) / 2.0
116 +
117 + if (mean < a - 2 * std) or (mean > b + 2 * std):
118 + warnings.warn(
119 + "mean is more than 2 std from [a, b] in nn.init.trunc_normal_. "
120 + "The distribution of values may be incorrect.",
121 + stacklevel=2,
122 + )
123 +
124 + # Values are generated by using a truncated uniform distribution and
125 + # then using the inverse CDF for the normal distribution.
126 + # Get upper and lower cdf values
127 + l_ = norm_cdf((a - mean) / std)
128 + u = norm_cdf((b - mean) / std)
129 +
130 + # Uniformly fill tensor with values from [l, u], then translate to
131 + # [2l-1, 2u-1].
132 + tensor.uniform_(2 * l_ - 1, 2 * u - 1)
133 +
134 + # Use inverse cdf transform for normal distribution to get truncated
135 + # standard normal
136 + if tensor.dtype in [torch.float16, torch.bfloat16]:
137 + # The `erfinv_` op is not (yet?) defined in float16+cpu, bfloat16+gpu
138 + og_dtype = tensor.dtype
139 + tensor = tensor.to(torch.float32)
140 + tensor.erfinv_()
141 + tensor = tensor.to(og_dtype)
142 + else:
143 + tensor.erfinv_()
144 +
145 + # Transform to proper mean, std
146 + tensor.mul_(std * math.sqrt(2.0))
147 + tensor.add_(mean)
148 +
149 + # Clamp to ensure it's in the proper range
150 + if tensor.dtype == torch.float16:
151 + # The `clamp_` op is not (yet?) defined in float16+cpu
152 + tensor = tensor.to(torch.float32)
153 + tensor.clamp_(min=a, max=b)
154 + tensor = tensor.to(torch.float16)
155 + else:
156 + tensor.clamp_(min=a, max=b)
157 +
158 +
159 +def trunc_normal_tf_(tensor: torch.Tensor,
160 + mean: float = 0.0,
161 + std: float = 1.0,
162 + a: float = -2.0,
163 + b: float = 2.0) -> torch.Tensor:
164 + with torch.no_grad():
165 + _trunc_normal_(tensor, 0, 1.0, a, b)
166 + tensor.mul_(std).add_(mean)
167 +
168 +
169 +def variance_scaling_(tensor, scale=1.0, mode="fan_in", distribution="normal"):
170 + fan_in, fan_out = _calculate_fan_in_and_fan_out(tensor)
171 + if mode == "fan_in":
172 + denom = fan_in
173 + elif mode == "fan_out":
174 + denom = fan_out
175 + elif mode == "fan_avg":
176 + denom = (fan_in + fan_out) / 2
177 +
178 + variance = scale / denom
179 +
180 + if distribution == "truncated_normal":
181 + # constant is stddev of standard normal truncated to (-2, 2)
182 + trunc_normal_tf_(tensor, std=math.sqrt(variance) / 0.87962566103423978)
183 + elif distribution == "normal":
184 + with torch.no_grad():
185 + tensor.normal_(std=math.sqrt(variance))
186 + elif distribution == "uniform":
187 + bound = math.sqrt(3 * variance)
188 + with torch.no_grad():
189 + tensor.uniform_(-bound, bound)
190 + else:
191 + raise ValueError(f"invalid distribution {distribution}")
192 +
193 +
194 +def lecun_normal_(tensor):
195 + variance_scaling_(tensor, mode="fan_in", distribution="truncated_normal")
196 +
197 +
198 +def default_flax_embed_init(tensor):
199 + variance_scaling_(tensor, mode="fan_in", distribution="normal")
200 +
201 +
202 +class SiglipVisionModelOutput(ModelOutput):
203 + image_embeds: Optional[torch.FloatTensor] = None
204 + last_hidden_state: torch.FloatTensor = None
205 + hidden_states: Optional[Tuple[torch.FloatTensor]] = None
206 + attentions: Optional[Tuple[torch.FloatTensor]] = None
207 +
208 +
209 +class SiglipVisionEmbeddings(nn.Module):
210 +
211 + def __init__(self, config: SiglipVisionConfig):
212 + super().__init__()
213 + self.config = config
214 + self.embed_dim = config.hidden_size
215 + self.image_size = config.image_size
216 + self.patch_size = config.patch_size
217 +
218 + self.patch_embedding = nn.Conv2d(
219 + in_channels=config.num_channels,
220 + out_channels=self.embed_dim,
221 + kernel_size=self.patch_size,
222 + stride=self.patch_size,
223 + padding="valid",
224 + )
225 +
226 + self.num_patches_per_side = self.image_size // self.patch_size
227 + self.num_patches = self.num_patches_per_side**2
228 + self.num_positions = self.num_patches
229 + self.position_embedding = nn.Embedding(self.num_positions,
230 + self.embed_dim)
231 +
232 + def forward(self,
233 + pixel_values: torch.FloatTensor,
234 + patch_attention_mask: torch.BoolTensor,
235 + tgt_sizes: Optional[torch.IntTensor] = None) -> torch.Tensor:
236 + batch_size = pixel_values.size(0)
237 +
238 + patch_embeds = self.patch_embedding(pixel_values)
239 + embeddings = patch_embeds.flatten(2).transpose(1, 2)
240 +
241 + max_im_h, max_im_w = pixel_values.size(2), pixel_values.size(3)
242 + max_nb_patches_h, max_nb_patches_w = (max_im_h // self.patch_size,
243 + max_im_w // self.patch_size)
244 + boundaries = torch.arange(1 / self.num_patches_per_side, 1.0,
245 + 1 / self.num_patches_per_side)
246 + position_ids = torch.full(
247 + size=(
248 + batch_size,
249 + max_nb_patches_h * max_nb_patches_w,
250 + ),
251 + fill_value=0,
252 + )
253 +
254 + for batch_idx, p_attn_mask in enumerate(patch_attention_mask):
255 + if tgt_sizes is not None:
256 + nb_patches_h = tgt_sizes[batch_idx][0]
257 + nb_patches_w = tgt_sizes[batch_idx][1]
258 + else:
259 + nb_patches_h = p_attn_mask[:, 0].sum()
260 + nb_patches_w = p_attn_mask[0].sum()
261 +
262 + fractional_coords_h = torch.arange(0, 1 - 1e-6, 1 / nb_patches_h)
263 + fractional_coords_w = torch.arange(0, 1 - 1e-6, 1 / nb_patches_w)
264 +
265 + bucket_coords_h = torch.bucketize(fractional_coords_h,
266 + boundaries,
267 + right=True)
268 + bucket_coords_w = torch.bucketize(fractional_coords_w,
269 + boundaries,
270 + right=True)
271 +
272 + pos_ids = (bucket_coords_h[:, None] * self.num_patches_per_side +
273 + bucket_coords_w).flatten()
274 + position_ids[batch_idx][p_attn_mask.view(-1).cpu()] = pos_ids
275 +
276 + position_ids = position_ids.to(self.position_embedding.weight.device)
277 +
278 + embeddings = embeddings + self.position_embedding(position_ids)
279 + return embeddings
280 +
281 +
282 +class SiglipAttention(nn.Module):
283 + """Multi-headed attention from 'Attention Is All You Need' paper"""
284 +
285 + # Copied from transformers.models.clip.modeling_clip.CLIPAttention.__init__
286 + def __init__(self, config):
287 + super().__init__()
288 + self.config = config
289 + self.embed_dim = config.hidden_size
290 + self.num_heads = config.num_attention_heads
291 + self.head_dim = self.embed_dim // self.num_heads
292 + if self.head_dim * self.num_heads != self.embed_dim:
293 + raise ValueError(
294 + "embed_dim must be divisible by num_heads (got `embed_dim`: "
295 + f"{self.embed_dim} and `num_heads`:"
296 + f" {self.num_heads}).")
297 + self.scale = self.head_dim**-0.5
298 + self.dropout = config.attention_dropout
299 +
300 + self.k_proj = nn.Linear(self.embed_dim, self.embed_dim)
301 + self.v_proj = nn.Linear(self.embed_dim, self.embed_dim)
302 + self.q_proj = nn.Linear(self.embed_dim, self.embed_dim)
303 + self.out_proj = nn.Linear(self.embed_dim, self.embed_dim)
304 +
305 + def forward(
306 + self,
307 + hidden_states: torch.Tensor,
308 + attention_mask: Optional[torch.Tensor] = None,
309 + output_attentions: Optional[bool] = False,
310 + ) -> Tuple[torch.Tensor, Optional[torch.Tensor],
311 + Optional[Tuple[torch.Tensor]]]:
312 + """Input shape: Batch x Time x Channel"""
313 +
314 + batch_size, q_len, _ = hidden_states.size()
315 +
316 + query_states = self.q_proj(hidden_states)
317 + key_states = self.k_proj(hidden_states)
318 + value_states = self.v_proj(hidden_states)
319 +
320 + query_states = query_states.view(batch_size, q_len, self.num_heads,
321 + self.head_dim).transpose(1, 2)
322 + key_states = key_states.view(batch_size, q_len, self.num_heads,
323 + self.head_dim).transpose(1, 2)
324 + value_states = value_states.view(batch_size, q_len, self.num_heads,
325 + self.head_dim).transpose(1, 2)
326 +
327 + k_v_seq_len = key_states.shape[-2]
328 + attn_weights = torch.matmul(query_states, key_states.transpose(
329 + 2, 3)) * self.scale
330 +
331 + if attn_weights.size() != (batch_size, self.num_heads, q_len,
332 + k_v_seq_len):
333 + raise ValueError(
334 + "Attention weights should be of size "
335 + f"{(batch_size, self.num_heads, q_len, k_v_seq_len)}, but is"
336 + f" {attn_weights.size()}")
337 +
338 + if attention_mask is not None:
339 + if attention_mask.size() != (batch_size, 1, q_len, k_v_seq_len):
340 + raise ValueError(
341 + "Attention mask should be of size "
342 + f"{(batch_size, 1, q_len, k_v_seq_len)}",
343 + f"but is {attention_mask.size()}")
344 + attn_weights = attn_weights + attention_mask
345 +
346 + # upcast attention to fp32
347 + attn_weights = nn.functional.softmax(attn_weights,
348 + dim=-1,
349 + dtype=torch.float32).to(
350 + query_states.dtype)
351 + attn_weights = nn.functional.dropout(attn_weights,
352 + p=self.dropout,
353 + training=self.training)
354 + attn_output = torch.matmul(attn_weights, value_states)
355 +
356 + if attn_output.size() != (batch_size, self.num_heads, q_len,
357 + self.head_dim):
358 + raise ValueError(
359 + "`attn_output` should be of size "
360 + f"{(batch_size, self.num_heads, q_len, self.head_dim)}, "
361 + "but is"
362 + f" {attn_output.size()}")
363 +
364 + attn_output = attn_output.transpose(1, 2).contiguous()
365 + attn_output = attn_output.reshape(batch_size, q_len, self.embed_dim)
366 +
367 + attn_output = self.out_proj(attn_output)
368 +
369 + return attn_output, attn_weights
370 +
371 +
372 +class SiglipFlashAttention2(SiglipAttention):
373 +
374 + def __init__(self, *args, **kwargs):
375 + super().__init__(*args, **kwargs)
376 + self.is_causal = False # Hack to make sure we don't use a causal mask
377 +
378 + def forward(
379 + self,
380 + hidden_states: torch.Tensor,
381 + attention_mask: Optional[torch.LongTensor] = None,
382 + position_ids: Optional[torch.LongTensor] = None,
383 + past_key_value: Optional[Tuple[torch.Tensor]] = None,
384 + output_attentions: bool = False,
385 + use_cache: bool = False,
386 + **kwargs,
387 + ) -> Tuple[torch.Tensor, Optional[torch.Tensor],
388 + Optional[Tuple[torch.Tensor]]]:
389 + output_attentions = False
390 +
391 + bsz, q_len, _ = hidden_states.size()
392 +
393 + query_states = self.q_proj(hidden_states)
394 + key_states = self.k_proj(hidden_states)
395 + value_states = self.v_proj(hidden_states)
396 +
397 + query_states = query_states.view(bsz, q_len, self.num_heads,
398 + self.head_dim).transpose(1, 2)
399 + key_states = key_states.view(bsz, q_len, self.num_heads,
400 + self.head_dim).transpose(1, 2)
401 + value_states = value_states.view(bsz, q_len, self.num_heads,
402 + self.head_dim).transpose(1, 2)
403 +
404 + kv_seq_len = key_states.shape[-2]
405 + if past_key_value is not None:
406 + kv_seq_len += past_key_value.get_usable_length(
407 + kv_seq_len, self.layer_idx)
408 +
409 + query_states = query_states.transpose(1, 2)
410 + key_states = key_states.transpose(1, 2)
411 + value_states = value_states.transpose(1, 2)
412 +
413 + dropout_rate = self.dropout if self.training else 0.0
414 +
415 + input_dtype = query_states.dtype
416 + if input_dtype == torch.float32:
417 + if torch.is_autocast_enabled():
418 + target_dtype = torch.get_autocast_gpu_dtype()
419 + # Handle the case where the model is quantized
420 + elif hasattr(self.config, "_pre_quantization_dtype"):
421 + target_dtype = self.config._pre_quantization_dtype
422 + else:
423 + target_dtype = self.q_proj.weight.dtype
424 +
425 + logger.warning(
426 + "The input hidden states seems to be "
427 + "silently casted in float32, "
428 + "this might be related to the fact "
429 + "you have upcasted embedding or layer norm layers in float32. "
430 + "We will cast back the input in"
431 + " %s.", target_dtype)
432 +
433 + query_states = query_states.to(target_dtype)
434 + key_states = key_states.to(target_dtype)
435 + value_states = value_states.to(target_dtype)
436 +
437 + attn_output = self._flash_attention_forward(query_states,
438 + key_states,
439 + value_states,
440 + attention_mask,
441 + q_len,
442 + dropout=dropout_rate)
443 +
444 + attn_output = attn_output.reshape(bsz, q_len,
445 + self.embed_dim).contiguous()
446 + attn_output = self.out_proj(attn_output)
447 +
448 + if not output_attentions:
449 + attn_weights = None
450 +
451 + return attn_output, attn_weights
452 +
453 + def _flash_attention_forward(self,
454 + query_states,
455 + key_states,
456 + value_states,
457 + attention_mask,
458 + query_length,
459 + dropout=0.0,
460 + softmax_scale=None):
461 + causal = self.is_causal and query_length != 1
462 +
463 + # Contains at least one padding token in the sequence
464 + if attention_mask is not None:
465 + batch_size = query_states.shape[0]
466 + (query_states, key_states, value_states, indices_q, cu_seq_lens,
467 + max_seq_lens) = self._upad_input(query_states, key_states,
468 + value_states, attention_mask,
469 + query_length)
470 +
471 + cu_seqlens_q, cu_seqlens_k = cu_seq_lens
472 + max_seqlen_in_batch_q, max_seqlen_in_batch_k = max_seq_lens
473 +
474 + attn_output_unpad = flash_attn_varlen_func(
475 + query_states,
476 + key_states,
477 + value_states,
478 + cu_seqlens_q=cu_seqlens_q,
479 + cu_seqlens_k=cu_seqlens_k,
480 + max_seqlen_q=max_seqlen_in_batch_q,
481 + max_seqlen_k=max_seqlen_in_batch_k,
482 + dropout_p=dropout,
483 + softmax_scale=softmax_scale,
484 + causal=causal,
485 + )
486 +
487 + attn_output = pad_input(attn_output_unpad, indices_q, batch_size,
488 + query_length)
489 + else:
490 + attn_output = flash_attn_func(query_states,
491 + key_states,
492 + value_states,
493 + dropout,
494 + softmax_scale=softmax_scale,
495 + causal=causal)
496 +
497 + return attn_output
498 +
499 + def _upad_input(self, query_layer, key_layer, value_layer, attention_mask,
500 + query_length):
501 + indices_k, cu_seqlens_k, max_seqlen_in_batch_k = _get_unpad_data(
502 + attention_mask)
503 + batch_size, kv_seq_len, num_key_value_heads, head_dim = key_layer.shape
504 +
505 + key_layer = index_first_axis(
506 + key_layer.reshape(batch_size * kv_seq_len, num_key_value_heads,
507 + head_dim), indices_k)
508 + value_layer = index_first_axis(
509 + value_layer.reshape(batch_size * kv_seq_len, num_key_value_heads,
510 + head_dim), indices_k)
511 + if query_length == kv_seq_len:
512 + query_layer = index_first_axis(
513 + query_layer.reshape(batch_size * kv_seq_len, self.num_heads,
514 + head_dim), indices_k)
515 + cu_seqlens_q = cu_seqlens_k
516 + max_seqlen_in_batch_q = max_seqlen_in_batch_k
517 + indices_q = indices_k
518 + elif query_length == 1:
519 + max_seqlen_in_batch_q = 1
520 + cu_seqlens_q = torch.arange(
521 + batch_size + 1, dtype=torch.int32, device=query_layer.device
522 + ) # There is a memcpy here, that is very bad.
523 + indices_q = cu_seqlens_q[:-1]
524 + query_layer = query_layer.squeeze(1)
525 + else:
526 + # The -q_len: slice assumes left padding.
527 + attention_mask = attention_mask[:, -query_length:]
528 + (query_layer, indices_q, cu_seqlens_q,
529 + max_seqlen_in_batch_q) = unpad_input(query_layer, attention_mask)
530 +
531 + return (
532 + query_layer,
533 + key_layer,
534 + value_layer,
535 + indices_q,
536 + (cu_seqlens_q, cu_seqlens_k),
537 + (max_seqlen_in_batch_q, max_seqlen_in_batch_k),
538 + )
539 +
540 +
541 +# Copied from transformers.models.clip.modeling_clip.CLIPMLP with CLIP->Siglip
542 +class SiglipMLP(nn.Module):
543 +
544 + def __init__(self, config):
545 + super().__init__()
546 + self.config = config
547 + self.activation_fn = ACT2FN[config.hidden_act]
548 + self.fc1 = nn.Linear(config.hidden_size, config.intermediate_size)
549 + self.fc2 = nn.Linear(config.intermediate_size, config.hidden_size)
550 +
551 + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
552 + hidden_states = self.fc1(hidden_states)
553 + hidden_states = self.activation_fn(hidden_states)
554 + hidden_states = self.fc2(hidden_states)
555 + return hidden_states
556 +
557 +
558 +# Copied from transformers.models.clip.modeling_clip.CLIPEncoderLayer
559 +# with CLIP->Siglip
560 +class SiglipEncoderLayer(nn.Module):
561 +
562 + def __init__(self, config: SiglipVisionConfig):
563 + super().__init__()
564 + self.embed_dim = config.hidden_size
565 + self._use_flash_attention_2 = (
566 + config._attn_implementation == "flash_attention_2")
567 + self.self_attn = (SiglipAttention(config)
568 + if not self._use_flash_attention_2 else
569 + SiglipFlashAttention2(config))
570 + self.layer_norm1 = nn.LayerNorm(self.embed_dim,
571 + eps=config.layer_norm_eps)
572 + self.mlp = SiglipMLP(config)
573 + self.layer_norm2 = nn.LayerNorm(self.embed_dim,
574 + eps=config.layer_norm_eps)
575 +
576 + def forward(
577 + self,
578 + hidden_states: torch.Tensor,
579 + attention_mask: torch.Tensor,
580 + output_attentions: Optional[bool] = False,
581 + ) -> Tuple[torch.FloatTensor]:
582 + residual = hidden_states
583 +
584 + hidden_states = self.layer_norm1(hidden_states)
585 + hidden_states, attn_weights = self.self_attn(
586 + hidden_states=hidden_states,
587 + attention_mask=attention_mask,
588 + output_attentions=output_attentions,
589 + )
590 + hidden_states = residual + hidden_states
591 +
592 + residual = hidden_states
593 + hidden_states = self.layer_norm2(hidden_states)
594 + hidden_states = self.mlp(hidden_states)
595 + hidden_states = residual + hidden_states
596 +
597 + outputs = (hidden_states, )
598 +
599 + if output_attentions:
600 + outputs += (attn_weights, )
601 +
602 + return outputs
603 +
604 +
605 +class SiglipPreTrainedModel(PreTrainedModel):
606 + config_class = SiglipVisionConfig
607 + base_model_prefix = "siglip"
608 + supports_gradient_checkpointing = True
609 +
610 + def _init_weights(self, module):
611 + """Initialize the weights"""
612 +
613 + if isinstance(module, SiglipVisionEmbeddings):
614 + width = self.config.hidden_size
615 + nn.init.normal_(module.position_embedding.weight,
616 + std=1 / np.sqrt(width))
617 + elif isinstance(module, nn.Embedding):
618 + default_flax_embed_init(module.weight)
619 + elif isinstance(module, SiglipAttention):
620 + nn.init.normal_(module.q_proj.weight)
621 + nn.init.normal_(module.k_proj.weight)
622 + nn.init.normal_(module.v_proj.weight)
623 + nn.init.normal_(module.out_proj.weight)
624 + nn.init.zeros_(module.q_proj.bias)
625 + nn.init.zeros_(module.k_proj.bias)
626 + nn.init.zeros_(module.v_proj.bias)
627 + nn.init.zeros_(module.out_proj.bias)
628 + elif isinstance(module, SiglipMLP):
629 + nn.init.normal_(module.fc1.weight)
630 + nn.init.normal_(module.fc2.weight)
631 + nn.init.normal_(module.fc1.bias, std=1e-6)
632 + nn.init.normal_(module.fc2.bias, std=1e-6)
633 + elif isinstance(module, (nn.Linear, nn.Conv2d)):
634 + lecun_normal_(module.weight)
635 + if module.bias is not None:
636 + nn.init.zeros_(module.bias)
637 + elif isinstance(module, nn.LayerNorm):
638 + module.bias.data.zero_()
639 + module.weight.data.fill_(1.0)
640 +
641 +
642 +# Copied from transformers.models.clip.modeling_clip.CLIPEncoder
643 +# with CLIP->Siglip
644 +class SiglipEncoder(nn.Module):
645 +
646 + def __init__(self, config: SiglipVisionConfig):
647 + super().__init__()
648 + self.config = config
649 + self.layers = nn.ModuleList([
650 + SiglipEncoderLayer(config) for _ in range(config.num_hidden_layers)
651 + ])
652 + self.gradient_checkpointing = False
653 +
654 + # Ignore copy
655 + def forward(
656 + self,
657 + inputs_embeds,
658 + attention_mask: Optional[torch.Tensor] = None,
659 + output_attentions: Optional[bool] = None,
660 + output_hidden_states: Optional[bool] = None,
661 + return_dict: Optional[bool] = None,
662 + ) -> Union[Tuple, BaseModelOutput]:
663 + output_attentions = output_attentions if output_attentions is not None \
664 + else self.config.output_attentions
665 + output_hidden_states = (output_hidden_states
666 + if output_hidden_states is not None else
667 + self.config.output_hidden_states)
668 + return_dict = return_dict if return_dict is not None \
669 + else self.config.use_return_dict
670 +
671 + encoder_states = () if output_hidden_states else None
672 + all_attentions = () if output_attentions else None
673 +
674 + hidden_states = inputs_embeds
675 + for encoder_layer in self.layers:
676 + if output_hidden_states:
677 + encoder_states = encoder_states + (hidden_states, )
678 + if self.gradient_checkpointing and self.training:
679 + layer_outputs = self._gradient_checkpointing_func(
680 + encoder_layer.__call__,
681 + hidden_states,
682 + attention_mask,
683 + output_attentions,
684 + )
685 + else:
686 + layer_outputs = encoder_layer(
687 + hidden_states,
688 + attention_mask,
689 + output_attentions=output_attentions,
690 + )
691 +
692 + hidden_states = layer_outputs[0]
693 +
694 + if output_attentions:
695 + all_attentions = all_attentions + (layer_outputs[1], )
696 +
697 + if output_hidden_states:
698 + encoder_states = encoder_states + (hidden_states, )
699 +
700 + if not return_dict:
701 + return tuple(
702 + v for v in [hidden_states, encoder_states, all_attentions]
703 + if v is not None)
704 + return BaseModelOutput(last_hidden_state=hidden_states,
705 + hidden_states=encoder_states,
706 + attentions=all_attentions)
707 +
708 +
709 +class SiglipVisionTransformer(SiglipPreTrainedModel):
710 + config_class = SiglipVisionConfig
711 + main_input_name = "pixel_values"
712 + _supports_flash_attn_2 = True
713 +
714 + def __init__(self, config: SiglipVisionConfig):
715 + super().__init__(config)
716 + self.config = config
717 + embed_dim = config.hidden_size
718 +
719 + self.embeddings = SiglipVisionEmbeddings(config)
720 + self.encoder = SiglipEncoder(config)
721 + self.post_layernorm = nn.LayerNorm(embed_dim,
722 + eps=config.layer_norm_eps)
723 + self._use_flash_attention_2 = (
724 + config._attn_implementation == "flash_attention_2")
725 +
726 + # Initialize weights and apply final processing
727 + self.post_init()
728 +
729 + def get_input_embeddings(self) -> nn.Module:
730 + return self.embeddings.patch_embedding
731 +
732 + @replace_return_docstrings(output_type=BaseModelOutputWithPooling,
733 + config_class=SiglipVisionConfig)
734 + def forward(
735 + self,
736 + pixel_values,
737 + patch_attention_mask: Optional[torch.BoolTensor] = None,
738 + tgt_sizes: Optional[torch.IntTensor] = None,
739 + output_attentions: Optional[bool] = None,
740 + output_hidden_states: Optional[bool] = None,
741 + return_dict: Optional[bool] = None,
742 + ) -> Union[Tuple, BaseModelOutputWithPooling]:
743 + r"""
744 + Returns:
745 + """
746 + output_attentions = output_attentions if output_attentions is not None \
747 + else self.config.output_attentions
748 + output_hidden_states = (output_hidden_states
749 + if output_hidden_states is not None else
750 + self.config.output_hidden_states)
751 + return_dict = return_dict if return_dict is not None \
752 + else self.config.use_return_dict
753 +
754 + batch_size = pixel_values.size(0)
755 + if patch_attention_mask is None:
756 + patch_attention_mask = torch.ones(
757 + size=(
758 + batch_size,
759 + pixel_values.size(2) // self.config.patch_size,
760 + pixel_values.size(3) // self.config.patch_size,
761 + ),
762 + dtype=torch.bool,
763 + device=pixel_values.device,
764 + )
765 +
766 + hidden_states = self.embeddings(
767 + pixel_values=pixel_values,
768 + patch_attention_mask=patch_attention_mask,
769 + tgt_sizes=tgt_sizes)
770 +
771 + patch_attention_mask = patch_attention_mask.view(batch_size, -1)
772 + # The call to `_upad_input` in `_flash_attention_forward` is expensive
773 + # So when the `patch_attention_mask` is full of 1s
774 + # (i.e. attending to the whole sequence),
775 + # avoiding passing the attention_mask,
776 + # which is equivalent to attending to the full sequence
777 + if not torch.any(~patch_attention_mask):
778 + attention_mask = None
779 + else:
780 + attention_mask = (_prepare_4d_attention_mask(
781 + patch_attention_mask, hidden_states.dtype)
782 + if not self._use_flash_attention_2 else
783 + patch_attention_mask)
784 +
785 + encoder_outputs = self.encoder(
786 + inputs_embeds=hidden_states,
787 + attention_mask=attention_mask,
788 + output_attentions=output_attentions,
789 + output_hidden_states=output_hidden_states,
790 + return_dict=return_dict,
791 + )
792 +
793 + last_hidden_state = encoder_outputs[0]
794 + last_hidden_state = self.post_layernorm(last_hidden_state)
795 +
796 + if not return_dict:
797 + return (last_hidden_state, None) + encoder_outputs[1:]
798 +
799 + return BaseModelOutputWithPooling(
800 + last_hidden_state=last_hidden_state,
801 + pooler_output=None,
802 + hidden_states=encoder_outputs.hidden_states,
803 + attentions=encoder_outputs.attentions,
804 + )
vllm/model_executor/models/qwen2.py
+1 -1
@@ -342,7 +342,7 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA):
342 intermediate_tensors: Optional[IntermediateTensors] = None,
343 ) -> torch.Tensor:
344 hidden_states = self.model(input_ids, positions, kv_caches,
345 - attn_metadata)
345 + attn_metadata, intermediate_tensors)
346 return hidden_states
347
348 def compute_logits(self, hidden_states: torch.Tensor,