@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d1f6d1c8af
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Model] Add has_weight to RMSNorm and re-enable weights loading tracker for Mamba (#10739)

Signed-off-by: Isotr0py <2037008807@qq.com>

Isotr0py committed Dec 10, 2024 at 10:23 UTC d1f6d1c8af892c7269f113711783374eebb52511
3 files changed +34 -12
vllm/model_executor/layers/layernorm.py
+9 -2
@@ -20,6 +20,7 @@ class RMSNorm(CustomOp):
20 hidden_size: int,
21 eps: float = 1e-6,
22 var_hidden_size: Optional[int] = None,
23 + has_weight: bool = True,
24 ) -> None:
25 super().__init__()
26
@@ -27,7 +28,11 @@ class RMSNorm(CustomOp):
28 self.variance_epsilon = eps
29 self.variance_size_override = (None if var_hidden_size == hidden_size
30 else var_hidden_size)
30 - self.weight = nn.Parameter(torch.ones(hidden_size))
31 + self.has_weight = has_weight
32 +
33 + self.weight = torch.ones(hidden_size)
34 + if self.has_weight:
35 + self.weight = nn.Parameter(self.weight)
36
37 def forward_native(
38 self,
@@ -59,7 +64,9 @@ class RMSNorm(CustomOp):
64 variance = x_var.pow(2).mean(dim=-1, keepdim=True)
65
66 x = x * torch.rsqrt(variance + self.variance_epsilon)
62 - x = x.to(orig_dtype) * self.weight
67 + x = x.to(orig_dtype)
68 + if self.has_weight:
69 + x = x * self.weight
70 if residual is None:
71 return x
72 else:
vllm/model_executor/layers/mamba/mamba_mixer.py
+18 -8
@@ -40,6 +40,7 @@ class MambaMixer(CustomOp):
40 use_conv_bias: bool,
41 use_bias: bool,
42 use_rms_norm: bool,
43 + rms_norm_has_weight: bool = True,
44 rms_norm_eps: float = 1e-5,
45 activation="silu"):
46 super().__init__()
@@ -105,14 +106,23 @@ class MambaMixer(CustomOp):
106 input_is_parallel=True,
107 )
108
108 - self.dt_layernorm = RMSNorm(time_step_rank,
109 - eps=rms_norm_eps) if use_rms_norm else None
110 -
111 - self.b_layernorm = RMSNorm(ssm_state_size,
112 - eps=rms_norm_eps) if use_rms_norm else None
113 -
114 - self.c_layernorm = RMSNorm(ssm_state_size,
115 - eps=rms_norm_eps) if use_rms_norm else None
109 + self.dt_layernorm = RMSNorm(
110 + time_step_rank,
111 + eps=rms_norm_eps,
112 + has_weight=rms_norm_has_weight,
113 + ) if use_rms_norm else None
114 +
115 + self.b_layernorm = RMSNorm(
116 + ssm_state_size,
117 + eps=rms_norm_eps,
118 + has_weight=rms_norm_has_weight,
119 + ) if use_rms_norm else None
120 +
121 + self.c_layernorm = RMSNorm(
122 + ssm_state_size,
123 + eps=rms_norm_eps,
124 + has_weight=rms_norm_has_weight,
125 + ) if use_rms_norm else None
126
127 def forward_native(self, hidden_states: torch.Tensor,
128 attn_metadata: AttentionMetadata,
vllm/model_executor/models/mamba.py
+7 -2
@@ -1,5 +1,5 @@
1 """PyTorch MAMBA model."""
2 -from typing import Iterable, List, Optional, Tuple
2 +from typing import Iterable, List, Optional, Set, Tuple
3
4 import torch
5 from torch import nn
@@ -47,6 +47,7 @@ class MambaDecoderLayer(nn.Module):
47 use_conv_bias=config.use_conv_bias,
48 use_bias=config.use_bias,
49 use_rms_norm=self.is_falcon_mamba,
50 + rms_norm_has_weight=not self.is_falcon_mamba,
51 rms_norm_eps=mixer_rms_eps,
52 activation=config.hidden_act)
53
@@ -241,8 +242,10 @@ class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree):
242 next_tokens = self.sampler(logits, sampling_metadata)
243 return next_tokens
244
244 - def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
245 + def load_weights(self, weights: Iterable[Tuple[str,
246 + torch.Tensor]]) -> Set[str]:
247 params_dict = dict(self.named_parameters())
248 + loaded_params: Set[str] = set()
249 for name, loaded_weight in weights:
250 if "A_log" in name:
251 name = name.replace("A_log", "A")
@@ -254,3 +257,5 @@ class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree):
257 weight_loader = getattr(param, "weight_loader",
258 default_weight_loader)
259 weight_loader(param, loaded_weight)
260 + loaded_params.add(name)
261 + return loaded_params