@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 65c08928c2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[Model] Remove unnecessary weight initialization logic (#11736)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Signed-off-by: Isotr0py <2037008807@qq.com> Co-authored-by: Isotr0py <2037008807@qq.com>

Cyrus Leung committed Jan 4, 2025 at 23:46 UTC 65c08928c2db934b18f7c6f5eeb02617826fae8e
3 files changed +5 -22
vllm/model_executor/layers/resampler.py
+4 -16
@@ -27,7 +27,7 @@
27 Shared resampler perceiver network used in multimodal models and
28 related helpers for sincos positional embeddings.
29
30 -Example models: Qwen (Qwen-VL), Minicpmv2.0
30 +Example models: Qwen (Qwen-VL), MiniCPM-V 2.0
31 """
32 import math
33 from functools import partial
@@ -37,7 +37,6 @@ import numpy as np
37 import torch
38 import torch.nn.functional as F
39 from torch import nn
40 -from torch.nn.init import trunc_normal_
40
41 from vllm.model_executor.layers.linear import ReplicatedLinear
42 from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -169,8 +168,8 @@ class BaseResampler(nn.Module):
168 self.embed_dim = embed_dim
169 self.num_heads = num_heads
170
172 - self.query = nn.Parameter(torch.zeros(self.num_queries, embed_dim))
173 - trunc_normal_(self.query, std=0.02)
171 + self.query = nn.Parameter(torch.empty(self.num_queries, embed_dim))
172 +
173 if kv_dim is not None and kv_dim != embed_dim:
174 self.kv_proj = ReplicatedLinear(kv_dim,
175 embed_dim,
@@ -190,16 +189,7 @@ class BaseResampler(nn.Module):
189 self.ln_post = norm_layer(embed_dim) if do_post_projection else None
190 self.proj = nn.Parameter(
191 (embed_dim**-0.5) *
193 - torch.randn(embed_dim, embed_dim)) if do_post_projection else None
194 -
195 - def _init_weights(self, m: nn.Module) -> None:
196 - if isinstance(m, nn.Linear):
197 - trunc_normal_(m.weight, std=0.02)
198 - if isinstance(m, nn.Linear) and m.bias is not None:
199 - nn.init.constant_(m.bias, 0)
200 - elif isinstance(m, nn.LayerNorm):
201 - nn.init.constant_(m.bias, 0)
202 - nn.init.constant_(m.weight, 1.0)
192 + torch.empty(embed_dim, embed_dim)) if do_post_projection else None
193
194 def _repeat(self, query, N: int):
195 return query.unsqueeze(1).repeat(1, N, 1)
@@ -240,8 +230,6 @@ class Resampler2(BaseResampler):
230 self.pos_embed = nn.Parameter(
231 torch.from_numpy(pos_embed_arr).requires_grad_(False))
232
243 - self.apply(self._init_weights)
244 -
233 def forward(
234 self,
235 x: torch.Tensor,
vllm/model_executor/models/aria.py
+1 -4
@@ -3,7 +3,6 @@ from typing import (Callable, Iterable, List, Mapping, Optional, Set, Tuple,
3
4 import torch
5 import torch.nn as nn
6 -from torch.nn.init import trunc_normal_
6 from transformers import BatchFeature, PretrainedConfig
7
8 from vllm.attention import AttentionMetadata
@@ -216,9 +215,7 @@ class AriaProjector(nn.Module):
215 self.num_heads = num_heads
216
217 self.query = nn.Parameter(
219 - torch.zeros(max(patch_to_query_dict.values()), self.embed_dim))
220 -
221 - trunc_normal_(self.query, std=0.02)
218 + torch.empty(max(patch_to_query_dict.values()), self.embed_dim))
219
220 self.cross_attn = CrossAttention(kv_dim, embed_dim, num_heads)
221
vllm/model_executor/models/minicpmv.py
-2
@@ -141,8 +141,6 @@ class Resampler2_5(BaseResampler):
141 self.max_size = max_size
142 self._set_2d_pos_cache(self.max_size)
143
144 - self.apply(self._init_weights)
145 -
144 def _set_2d_pos_cache(self,
145 max_size: Tuple[int, int],
146 device: torch.types.Device = "cpu") -> None: