A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[Bugfix] Fixup Mamba (#10004)
Signed-off-by: Tyler Michael Smith <tyler@neuralmagic.com>
Tyler Michael Smith committed
Nov 4, 2024 at 22:46 UTC
ad23318928d40ef7ac969451afa0dc198428c04b
1 file changed
+3
-4
vllm/model_executor/models/mamba.py
+3
-4
@@ -39,8 +39,8 @@ class MambaDecoderLayer(nn.Module):
39
super().__init__()
40
self.config = config
41
self.is_falcon_mamba = config.model_type == "falcon_mamba"
42
- mixer_rms_rps = config.mixer_rms_rps if self.is_falcon_mamba else None
43
- self.mamba = MambaMixer(hidden_size=config.hidden_size,
42
+ mixer_rms_eps = config.mixer_rms_eps if self.is_falcon_mamba else None
43
+ self.mixer = MambaMixer(hidden_size=config.hidden_size,
44
ssm_state_size=config.state_size,
45
conv_kernel_size=config.conv_kernel,
46
intermediate_size=config.intermediate_size,
@@ -48,7 +48,7 @@ class MambaDecoderLayer(nn.Module):
48
use_conv_bias=config.use_conv_bias,
49
use_bias=config.use_bias,
50
use_rms_norm=self.is_falcon_mamba,
51
- rms_norm_eps=mixer_rms_rps,
51
+ rms_norm_eps=mixer_rms_eps,
52
activation=config.hidden_act)
53
54
self.norm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
@@ -99,7 +99,6 @@ class MambaModel(nn.Module):
99
for i in range(config.num_hidden_layers):
100
decoder_layers.append(
101
MambaDecoderLayer(config,
102
- layer_idx=i,
102
cache_config=cache_config,
103
quant_config=quant_config))
104
self.layers = nn.ModuleList(decoder_layers)