A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[Model] Remove unnecessary weight initialization logic (#11736)
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk> Signed-off-by: Isotr0py <2037008807@qq.com> Co-authored-by: Isotr0py <2037008807@qq.com>
Cyrus Leung committed
Jan 4, 2025 at 23:46 UTC
65c08928c2db934b18f7c6f5eeb02617826fae8e
3 files changed
+5
-22
vllm/model_executor/layers/resampler.py
+4
-16
@@ -27,7 +27,7 @@
27
Shared resampler perceiver network used in multimodal models and
28
related helpers for sincos positional embeddings.
29
30
-Example models: Qwen (Qwen-VL), Minicpmv2.0
30
+Example models: Qwen (Qwen-VL), MiniCPM-V 2.0
31
"""
32
import math
33
from functools import partial
@@ -37,7 +37,6 @@ import numpy as np
37
import torch
38
import torch.nn.functional as F
39
from torch import nn
40
-from torch.nn.init import trunc_normal_
40
41
from vllm.model_executor.layers.linear import ReplicatedLinear
42
from vllm.model_executor.layers.quantization import QuantizationConfig
@@ -169,8 +168,8 @@ class BaseResampler(nn.Module):
168
self.embed_dim = embed_dim
169
self.num_heads = num_heads
170
172
- self.query = nn.Parameter(torch.zeros(self.num_queries, embed_dim))
173
- trunc_normal_(self.query, std=0.02)
171
+ self.query = nn.Parameter(torch.empty(self.num_queries, embed_dim))
172
+
173
if kv_dim is not None and kv_dim != embed_dim:
174
self.kv_proj = ReplicatedLinear(kv_dim,
175
embed_dim,
@@ -190,16 +189,7 @@ class BaseResampler(nn.Module):
189
self.ln_post = norm_layer(embed_dim) if do_post_projection else None
190
self.proj = nn.Parameter(
191
(embed_dim**-0.5) *
193
- torch.randn(embed_dim, embed_dim)) if do_post_projection else None
194
-
195
- def _init_weights(self, m: nn.Module) -> None:
196
- if isinstance(m, nn.Linear):
197
- trunc_normal_(m.weight, std=0.02)
198
- if isinstance(m, nn.Linear) and m.bias is not None:
199
- nn.init.constant_(m.bias, 0)
200
- elif isinstance(m, nn.LayerNorm):
201
- nn.init.constant_(m.bias, 0)
202
- nn.init.constant_(m.weight, 1.0)
192
+ torch.empty(embed_dim, embed_dim)) if do_post_projection else None
193
194
def _repeat(self, query, N: int):
195
return query.unsqueeze(1).repeat(1, N, 1)
@@ -240,8 +230,6 @@ class Resampler2(BaseResampler):
230
self.pos_embed = nn.Parameter(
231
torch.from_numpy(pos_embed_arr).requires_grad_(False))
232
243
- self.apply(self._init_weights)
244
-
233
def forward(
234
self,
235
x: torch.Tensor,
vllm/model_executor/models/aria.py
+1
-4
@@ -3,7 +3,6 @@ from typing import (Callable, Iterable, List, Mapping, Optional, Set, Tuple,
3
4
import torch
5
import torch.nn as nn
6
-from torch.nn.init import trunc_normal_
6
from transformers import BatchFeature, PretrainedConfig
7
8
from vllm.attention import AttentionMetadata
@@ -216,9 +215,7 @@ class AriaProjector(nn.Module):
215
self.num_heads = num_heads
216
217
self.query = nn.Parameter(
219
- torch.zeros(max(patch_to_query_dict.values()), self.embed_dim))
220
-
221
- trunc_normal_(self.query, std=0.02)
218
+ torch.empty(max(patch_to_query_dict.values()), self.embed_dim))
219
220
self.cross_attn = CrossAttention(kv_dim, embed_dim, num_heads)
221
vllm/model_executor/models/minicpmv.py
-2
@@ -141,8 +141,6 @@ class Resampler2_5(BaseResampler):
141
self.max_size = max_size
142
self._set_2d_pos_cache(self.max_size)
143
144
- self.apply(self._init_weights)
145
-
144
def _set_2d_pos_cache(self,
145
max_size: Tuple[int, int],
146
device: torch.types.Device = "cpu") -> None: