@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ad23318928
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[Bugfix] Fixup Mamba (#10004)

Signed-off-by: Tyler Michael Smith <tyler@neuralmagic.com>

Tyler Michael Smith committed Nov 4, 2024 at 22:46 UTC ad23318928d40ef7ac969451afa0dc198428c04b
1 file changed +3 -4
vllm/model_executor/models/mamba.py
+3 -4
@@ -39,8 +39,8 @@ class MambaDecoderLayer(nn.Module):
39 super().__init__()
40 self.config = config
41 self.is_falcon_mamba = config.model_type == "falcon_mamba"
42 - mixer_rms_rps = config.mixer_rms_rps if self.is_falcon_mamba else None
43 - self.mamba = MambaMixer(hidden_size=config.hidden_size,
42 + mixer_rms_eps = config.mixer_rms_eps if self.is_falcon_mamba else None
43 + self.mixer = MambaMixer(hidden_size=config.hidden_size,
44 ssm_state_size=config.state_size,
45 conv_kernel_size=config.conv_kernel,
46 intermediate_size=config.intermediate_size,
@@ -48,7 +48,7 @@ class MambaDecoderLayer(nn.Module):
48 use_conv_bias=config.use_conv_bias,
49 use_bias=config.use_bias,
50 use_rms_norm=self.is_falcon_mamba,
51 - rms_norm_eps=mixer_rms_rps,
51 + rms_norm_eps=mixer_rms_eps,
52 activation=config.hidden_act)
53
54 self.norm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
@@ -99,7 +99,6 @@ class MambaModel(nn.Module):
99 for i in range(config.num_hidden_layers):
100 decoder_layers.append(
101 MambaDecoderLayer(config,
102 - layer_idx=i,
102 cache_config=cache_config,
103 quant_config=quant_config))
104 self.layers = nn.ModuleList(decoder_layers)